diff --git a/.github/workflows/poetry_update.yml b/.github/workflows/auto_updates.yml similarity index 60% rename from .github/workflows/poetry_update.yml rename to .github/workflows/auto_updates.yml index 9e02bad4..bad2461c 100644 --- a/.github/workflows/poetry_update.yml +++ b/.github/workflows/auto_updates.yml @@ -1,5 +1,6 @@ # This is a workflow for updating Python dependencies with Poetry. # Major version updates are handled separately, by Dependabot. +# It will also update the pre-commit hooks to use latest tags. --- name: Update Deps @@ -10,8 +11,8 @@ on: - cron: '35 14 * * 1' jobs: - poetry-update: - name: Update Python dependencies + workflow-auto-updates: + name: Update dependencies and hooks runs-on: ubuntu-latest strategy: matrix: @@ -24,8 +25,19 @@ jobs: - name: Checkout the repo uses: actions/checkout@v3 - - name: Install poetry - run: pipx install poetry + # This GPG key is for the `phylum-bot` account and used in order to ensure commits are signed/verified + - name: Import GPG key for bot account + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.PHYLUM_BOT_GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PHYLUM_BOT_GPG_PASSPHRASE }} + git_user_signingkey: true + git_commit_gpgsign: true + + - name: Install poetry and pre-commit + run: | + pipx install poetry + pipx install pre-commit - name: Configure poetry run: poetry config virtualenvs.in-project true @@ -45,17 +57,20 @@ jobs: poetry env use python${{ matrix.python-version }} poetry install --verbose --no-root - - name: Poetry update + - name: Update Python dependencies run: poetry update -vv + - name: Update pre-commit hooks + run: pre-commit autoupdate --freeze + - name: Commit changes id: commit continue-on-error: true + # NOTE: The git user name and email used for commits is already configured, + # by the crazy-max/ghaction-import-gpg action. run: | - git config user.name 'Phylum Bot' - git config user.email 'phylum-bot@users.noreply.github.com' - git commit -a -m "build: Bump poetry.lock dependencies" - git push --force origin HEAD:auto-poetry-update + git commit -a -m "build: Bump poetry.lock dependencies and pre-commit hooks" + git push --force origin HEAD:workflow-auto-updates - name: Create Pull Request if: ${{ steps.commit.outcome == 'success' }} @@ -66,8 +81,8 @@ jobs: github.rest.pulls.create({ owner: context.repo.owner, repo: context.repo.repo, - head: "auto-poetry-update", + head: "workflow-auto-updates", base: context.ref, - title: "build: Bump poetry.lock dependencies", - body: "Bump dependencies in poetry.lock for all SemVer-compatible updates.", + title: "build: Bump poetry.lock dependencies and pre-commit hooks", + body: "Bump dependencies in `poetry.lock` and hooks in `.pre-commit-config.yaml`.", }); diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..e8454ef3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +# This is the config for using `pre-commit` on this repository. +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +# +# NOTE: Individual hook revisions are kept up to date automatically with +# the `auto_updates` workflow, which bumps hooks to the latest tag. +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3298ddab3c13dd77d6ce1fc0baf97691430d84b0 # frozen: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: f6c139c5215ce04fd3e73a900f1372942d58eca0 # frozen: 22.6.0 + hooks: + - id: black + + - repo: https://github.com/asottile/pyupgrade + rev: a78007c1e9de96e71d5fb3e720c2b9fae8ed8abf # frozen: v2.37.3 + hooks: + - id: pyupgrade + args: [--py37-plus] + + # NOTE: don't use this config for your own repositories. Instead, see + # "Git pre-commit Integration" in `docs/sync/git_precommit.md` + - repo: local + hooks: + - id: phylum-ci + name: analyze lockfile with phylum-ci + language: system + files: ^poetry\.lock$ + entry: poetry run phylum-ci diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 00000000..41cc80cc --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,11 @@ +# This is the config for defining `pre-commit` hooks for use in other repositories. +# +# See https://pre-commit.com for more information +--- +- id: phylum + name: analyze lockfile with phylum + description: Run `phylum` on a dependency lockfile + entry: phylum-ci + language: python + require_serial: true + stages: [commit] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index badce5cf..05e60356 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,20 @@ Here's how to set up `phylum-ci` for local development. git clone git@github.com:phylum-dev/phylum-ci.git ``` -2. Ensure all supported Python versions are installed locally +2. Optional: Install [pre-commit](https://pre-commit.com/) and the local hooks + + ```sh + # If the `pre-commit` tool is not already installed, the recommended method is to use pipx + pipx install pre-commit + + # Installing with homebrew is another good option + brew install pre-commit + + # Use the `pre-commit` tool to install the git hooks used by the repository + pre-commit install + ``` + +3. Ensure all supported Python versions are installed locally 1. The strategy is to support all released minor versions of Python that are not end-of-life yet 2. The current list 1. at the time of this writing is 3.7, 3.8, 3.9, and 3.10 @@ -108,15 +121,15 @@ Here's how to set up `phylum-ci` for local development. pyenv global 3.10.x 3.9.x 3.8.x 3.7.x ``` -3. Ensure [poetry](https://python-poetry.org/docs/) is installed -4. Install dependencies with `poetry`, which will automatically create a virtual environment: +4. Ensure [poetry](https://python-poetry.org/docs/) is installed +5. Install dependencies with `poetry`, which will automatically create a virtual environment: ```sh cd phylum-ci poetry install ``` -5. Create a branch for local development: +6. Create a branch for local development: ```sh git checkout -b @@ -124,35 +137,40 @@ Here's how to set up `phylum-ci` for local development. Now you can make your changes locally. -6. If new dependencies are added, do so in a way that does not add upper version constraints and ensure +7. If new dependencies are added, do so in a way that does not add upper version constraints and ensure the `poetry.lock` file is updated (and committed): ```sh # Unless there is a reason to do so, prefer to add dependencies without constraints - poetry add new-dependency-name + poetry add "new-dependency-name==*" - # When a version constraint is not specified, poetry chooses one. For example (in pyproject.toml): + # When a version constraint is not specified, poetry chooses one. For example, the command: + # + # $ poetry add new-dependency-name + # + # results in a caret-style version constraint added to the dependency in pyproject.toml: # # new-dependency-name = "^1.2.3" # - # Unless the constraint was intentional, change the entry to remove the constraint: + # Unless the constraint was intentional, change the pyproject.toml entry to remove the constraint: # # new-dependency-name = "*" # Update the lockfile and the local environment to get the latest versions of dependencies poetry update - # Dependencies will be checked automatically in CI during a PR, but checking locally is possible: + # Dependencies will be checked automatically in CI during a PR. They will also be checked + # with the local pre-commit hook, if enabled. Manually checking locally is also possible: phylum analyze poetry.lock ``` -7. When you're done making changes, check that your changes pass the tests: +8. When you're done making changes, check that your changes pass the tests: ```sh poetry run tox ``` -8. Commit your changes and push your branch to GitHub: +9. Commit your changes and push your branch to GitHub: ```sh git add . @@ -160,7 +178,7 @@ Here's how to set up `phylum-ci` for local development. git push --set-upstream origin ``` -9. Submit a pull request (PR) through the GitHub website +10. Submit a pull request (PR) through the GitHub website ## Pull Request Guidelines diff --git a/README.md b/README.md index aa386995..9cdc9122 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ ![GitHub last commit](https://img.shields.io/github/last-commit/phylum-dev/phylum-ci) [![GitHub Workflow Status (branch)][workflow_shield]][workflow_test] [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)][CoC] +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)][pre-commit] Utilities for integrating Phylum into CI pipelines (and beyond) @@ -15,6 +16,7 @@ Utilities for integrating Phylum into CI pipelines (and beyond) [workflow_shield]: https://img.shields.io/github/workflow/status/phylum-dev/phylum-ci/Test/main?label=Test&logo=GitHub [workflow_test]: https://github.com/phylum-dev/phylum-ci/actions/workflows/test.yml [CoC]: https://github.com/phylum-dev/phylum-ci/blob/main/CODE_OF_CONDUCT.md +[pre-commit]: https://github.com/pre-commit/pre-commit [contributing]: https://github.com/phylum-dev/phylum-ci/blob/main/CONTRIBUTING.md [changelog]: https://github.com/phylum-dev/phylum-ci/blob/main/CHANGELOG.md [security]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/security.md @@ -116,14 +118,18 @@ The current CI platforms/environments supported are: * GitHub Actions * See the [GitHub Actions Integration documentation][github_docs] for more info +* Git `pre-commit` Hooks + * See the [Git `pre-commit` Integration documentation][precommit_docs] for more info + * None (local use) * This is the "fall-through" case used when no other environment is detected * Can be useful to analyze lockfiles locally, prior to or after submitting a pull/merge request (PR/MR) to a CI system * Establishing a successful submission prior to submitting a PR/MR to a CI system * Troubleshooting after submitting a PR/MR to a CI system and getting unexpected results -[gitlab_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/gitlab_ci.md -[github_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/github_actions.md +[gitlab_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/gitlab_ci.md +[github_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/github_actions.md +[precommit_docs]: https://github.com/phylum-dev/phylum-ci/blob/main/docs/sync/git_precommit.md ## License diff --git a/docs/sync/git_precommit.md b/docs/sync/git_precommit.md new file mode 100644 index 00000000..0b1ea1a2 --- /dev/null +++ b/docs/sync/git_precommit.md @@ -0,0 +1,162 @@ +--- +title: Git pre-commit Integration +category: 62cdf6722c2c1602a4b69643 +hidden: false +--- +# Git `pre-commit` Integration + +## Overview + +[pre-commit] is a framework for managing and maintaining multi-language Git pre-commit hooks. + +[pre-commit]: https://pre-commit.com/ + +Phylum is available as a pre-commit hook. + +Once configured for a repository, the git `pre-commit` integration will provide analysis of project dependencies +from a lockfile during a commit containing that lockfile. The hook will fail and provide a report if any of the +newly added/modified dependencies from the commit fail to meet the project risk thresholds for any of the five +Phylum risk domains: + +* Vulnerability (aka `vul`) +* Malicious Code (aka `mal`) +* Engineering (aka `eng`) +* License (aka `lic`) +* Author (aka `aut`) + +See [Phylum Risk Domains documentation](https://docs.phylum.io/docs/phylum-package-score#risk-domains) for more detail. + +**NOTE**: It is not enough to have the total project threshold set. Individual risk domain threshold values +must be set, either in the Phylum web UI or with `phylum-ci` options, in order to enable analysis results. +Otherwise, the risk domain is considered disabled and the threshold value used will be zero (0). + +The hook will be skipped if no dependencies were added or modified for a given commit. +If one or more dependencies are still processing (no results available), then the hook will only fail if +dependencies that have _completed analysis results_ do not meet the specified project risk thresholds. + +## Prerequisites + +The pre-requisites for using the git `pre-commit` hook are: + +* The [pre-commit] package manager installed +* A [Phylum token][phylum_tokens] with API access + * [Contact Phylum][phylum_contact] or create an account and register to gain access + * See also [`phylum auth register`][phylum_register] command documentation + * Consider using a bot or group account for this token +* Access to the Phylum API endpoints + * That usually means a connection to the internet, optionally via a proxy + * Support for on-premises installs are not available at this time +* A `.phylum_project` file exists at the root of the repository + * See [`phylum project`](https://docs.phylum.io/docs/phylum_project) and + [`phylum project create`](https://docs.phylum.io/docs/phylum_project_create) command documentation + +[phylum_tokens]: https://docs.phylum.io/docs/api-keys +[phylum_contact]: https://phylum.io/contact-us/ +[phylum_register]: https://docs.phylum.io/docs/phylum_auth_register + +**NOTE: If the `phylum` CLI binary is installed locally, it will be used. Otherwise, the hook will install it.** + +## Configure `.pre-commit-config.yaml` + +Phylum analysis of dependencies can be added to existing `pre-commit` configurations or +on it's own with this minimal configuration: + +```yaml +# This is the config for using `pre-commit` on this repository. +# +# See https://pre-commit.com for more information +--- +repos: + - repo: https://github.com/phylum-dev/phylum-ci + rev: main + hooks: + - id: phylum + # Optional: Specify the lockfile pattern for your repository + files: '' + # Optional: Specify additional arguments to be passed to `phylum-ci` + args: [] +``` + +**NOTE**: This example configuration uses a mutable reference for `rev`, which is a bad practice +(and only done here to prevent old tags from being used through copy and paste implementations). +A best practice is to ensure the `rev` key for all hooks is updated to a valid and current immutable reference: + +```sh +pre-commit autoupdate --freeze +``` + +The hook can be customized with [optional keys][hook_config] in the config file. +Two common customization keys for the `phylum` hook are `files` and `args`: + +[hook_config]: https://pre-commit.com/index.html#pre-commit-configyaml---hooks + +### File Control + +The `files` key in the hook configuration file is the way to ensure the hook only runs when the specified +lockfile has changed, saving execution time. + +The value for the `files` key is a [Python regular expression][re] and are matched with `re.search`. + +[re]: https://docs.python.org/3/library/re.html#regular-expression-syntax + +```yaml + # NOTE: These are examples. Only one `files` key for the hook is expected + + # Specify `package-lock.json` + files: ^package-lock\.json$ + + # Specify `poetry.lock` + files: ^poetry\.lock$ + + # Specify `requirements-*.txt` files + # NOTE: An explicit lockfile should still be specified in the `args` key + files: ^requirements-.*\.txt$ +``` + +### Argument Control + +The `args` key is the way to exert control over the execution of the Phylum analysis. +The `phylum-ci` script entry point is called by the hook. It has a number of arguments that are all optional +and defaulted to secure values. To view the arguments, their description, and default values, run the script +with `--help` output as specified in the [Usage section of the top-level README.md][usage] or view the +[source code][src] directly. + +[usage]: https://github.com/phylum-dev/phylum-ci/blob/main/README.md#usage +[src]: https://github.com/phylum-dev/phylum-ci/blob/main/src/phylum/ci/cli.py + +```yaml + # NOTE: These are examples. Only one `args` key for the hook is expected + + # Use the defaults for all the arguments. + # The default behavior is to only analyze newly added dependencies against + # the risk domain threshold levels set at the Phylum project level. + # The key can be removed if the defaults are used. + args: [] + + # Consider all dependencies in analysis results instead of just the newly added ones. + # The default is to only analyze newly added dependencies, which can be useful for + # existing code bases that may not meet established project risk thresholds yet, + # but don't want to make things worse. Specifying `--all-deps` can be useful for + # casting the widest net for strict adherence to Quality Assurance (QA) standards. + args: [--all-deps] + + # Some lockfile types (e.g., Python/pip `requirements.txt`) are ambiguous in that + # they can be named differently and may or may not contain strict dependencies. + # In these cases, it is best to specify an explicit lockfile path. + args: [--lockfile=requirements-prod.txt] + + # Thresholds for the five risk domains may be set at the Phylum project level. + # They can be set differently for the hook. + # NOTE: The shortened form is used here for brevity, but the long form might be more + # descriptive for future readers. For instance `--vul-threshold` instead of `-u`. + args: [-u=60, -m=60, -e=70, -c=90, -o=80] + + # Ensure the latest Phylum CLI is installed. + args: [--force-install] + + # Install a specific version of the Phylum CLI. + args: [--phylum-release=3.3.0, --force-install] + + # Mix and match for your specific use case. + args: [-u=60, -m=60, -e=70, -c=90, -o=80, --lockfile=requirements-prod.txt, --all-deps] +``` diff --git a/docs/sync/integrations_overview.md b/docs/sync/integrations_overview.md index 434df061..b47a09c6 100644 --- a/docs/sync/integrations_overview.md +++ b/docs/sync/integrations_overview.md @@ -24,6 +24,12 @@ See the [GitLab CI Integration documentation][gitlab_docs] for more info. [gitlab_docs]: https://docs.phylum.io/docs/gitlab_ci +### Git `pre-commit` Hooks + +See the [Git `pre-commit` documentation][precommit_docs] for more info. + +[precommit_docs]: https://docs.phylum.io/docs/git_precommit + ## Future Integrations If there is an unsupported use case for managing the security of your dependencies, we want to know about it. diff --git a/poetry.lock b/poetry.lock index 3cb7538b..b68ea7f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,6 +97,29 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "connect-markdown-renderer" +version = "2.0.1" +description = "Connect Markdown Renderer" +category = "main" +optional = false +python-versions = ">=3.7,<4" + +[package.dependencies] +markdown-it-py = ">=2.1.0,<3.0.0" +rich = ">=12.4.1,<13.0.0" + [[package]] name = "cryptography" version = "37.0.4" @@ -261,6 +284,36 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "markdown-it-py" +version = "2.1.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +mdurl = ">=0.1,<1.0" +typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[package.extras] +testing = ["pytest-regressions", "pytest-cov", "pytest", "coverage"] +rtd = ["sphinx-book-theme", "sphinx-design", "sphinx-copybutton", "sphinx", "pyyaml", "myst-parser", "attrs"] +profiling = ["gprof2dot"] +plugins = ["mdit-py-plugins"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +compare = ["panflute (>=2.1.3,<2.2.0)", "mistune (>=2.0.2,<2.1.0)", "mistletoe (>=0.8.1,<0.9.0)", "markdown (>=3.3.6,<3.4.0)", "commonmark (>=0.9.1,<0.10.0)"] +code_style = ["pre-commit (==2.6)"] +benchmarking = ["pytest-benchmark (>=3.2,<4.0)", "pytest", "psutil"] + +[[package]] +name = "mdurl" +version = "0.1.1" +description = "Markdown URL utilities" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "packaging" version = "21.3" @@ -330,7 +383,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pygments" version = "2.12.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -485,6 +538,22 @@ python-versions = ">=3.7" [package.extras] idna2008 = ["idna"] +[[package]] +name = "rich" +version = "12.5.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + [[package]] name = "ruamel.yaml" version = "0.17.21" @@ -724,7 +793,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "27b1c1c9fddbf146d091a57659d47f39e7637deafcb77f380e001271d007c1d1" +content-hash = "22dcaa94a88956d99d31b680a2befe9781ae7033cca5d4d44c6f312bba3d5c39" [metadata.files] atomicwrites = [] @@ -742,6 +811,14 @@ click-log = [ {file = "click_log-0.4.0-py2.py3-none-any.whl", hash = "sha256:a43e394b528d52112af599f2fc9e4b7cf3c15f94e53581f74fa6867e68c91756"}, ] colorama = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +connect-markdown-renderer = [ + {file = "connect-markdown-renderer-2.0.1.tar.gz", hash = "sha256:fbaefff195a6c347a7f89e75a09e78f6be35c903d72a2766c8f263ca75758757"}, + {file = "connect_markdown_renderer-2.0.1-py3-none-any.whl", hash = "sha256:8d726b947d877697a42f528799908481685bf0db6a17b4e6d715389bfae9cccd"}, +] cryptography = [] distlib = [] docutils = [] @@ -771,6 +848,14 @@ jeepney = [ {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, ] keyring = [] +markdown-it-py = [ + {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, + {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, +] +mdurl = [ + {file = "mdurl-0.1.1-py3-none-any.whl", hash = "sha256:6a8f6804087b7128040b2fb2ebe242bdc2affaeaa034d5fc9feeed30b443651b"}, + {file = "mdurl-0.1.1.tar.gz", hash = "sha256:f79c9709944df218a4cdb0fcc0b0c7ead2f44594e3e84dc566606f04ad749c20"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -818,6 +903,10 @@ rfc3986 = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, ] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] "ruamel.yaml" = [ {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, diff --git a/pyproject.toml b/pyproject.toml index b0d06d7e..0838ff23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ requests = "*" cryptography = "*" packaging = "*" "ruamel.yaml" = "*" +connect-markdown-renderer = "*" [tool.poetry.dev-dependencies] pytest = "*" diff --git a/src/phylum/ci/ci_base.py b/src/phylum/ci/ci_base.py index c231760e..f10cad95 100644 --- a/src/phylum/ci/ci_base.py +++ b/src/phylum/ci/ci_base.py @@ -64,6 +64,20 @@ def __init__(self, args: Namespace) -> None: self.args = args self._phylum_project_file = Path.cwd().joinpath(".phylum_project").resolve() + # The lockfile specified as a script argument will be used, if provided. + # Otherwise, an attempt will be made to automatically detect the lockfile. + provided_lockfile: Path = args.lockfile + if provided_lockfile and provided_lockfile.exists() and provided_lockfile.stat().st_size: + self._lockfile = provided_lockfile.resolve() + else: + detected_lockfile = detect_lockfile() + if detected_lockfile: + self._lockfile = detected_lockfile + else: + raise SystemExit( + " [!] A lockfile is required and was not detected. Consider specifying one with `--lockfile`." + ) + # Ensure all pre-requisites are met and bail at the earliest opportunity when they aren't self._check_prerequisites() print(" [+] All pre-requisites met") @@ -82,20 +96,6 @@ def __init__(self, args: Namespace) -> None: os.environ[TOKEN_ENVVAR_NAME] = args.token self.args.token = token - # The lockfile specified as a script argument will be used, if provided. - # Otherwise, an attempt will be made to automatically detect the lockfile. - provided_lockfile: Path = args.lockfile - if provided_lockfile and provided_lockfile.exists() and provided_lockfile.stat().st_size: - self._lockfile = provided_lockfile.resolve() - else: - detected_lockfile = detect_lockfile() - if detected_lockfile: - self._lockfile = detected_lockfile - else: - raise SystemExit( - " [!] A lockfile is required and was not detected. Consider specifying one with `--lockfile`." - ) - self._lockfile_changed = self._is_lockfile_changed(self.lockfile) @property @@ -120,7 +120,7 @@ def cli_path(self) -> Optional[Path]: @property def analysis_output(self) -> str: - """Get the output from the overall analysis.""" + """Get the output from the overall analysis, in markdown format.""" return self._analysis_output @property diff --git a/src/phylum/ci/ci_none.py b/src/phylum/ci/ci_none.py index 8b63a661..b75b1cdc 100644 --- a/src/phylum/ci/ci_none.py +++ b/src/phylum/ci/ci_none.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Optional -from phylum.ci import SCRIPT_NAME +from connect.utils.terminal.markdown import render from phylum.ci.ci_base import CIBase @@ -60,7 +60,7 @@ def phylum_label(self) -> str: # Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects cmd = f"git hash-object {self.lockfile}".split() lockfile_hash_object = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip() - label = f"{SCRIPT_NAME}_{self.ci_platform_name}_{current_branch}_{lockfile_hash_object}" + label = f"{self.ci_platform_name}_{current_branch}_{lockfile_hash_object[:7]}" label = label.replace(" ", "-") return label @@ -104,6 +104,4 @@ def _is_lockfile_changed(self, lockfile: Path) -> bool: def post_output(self) -> None: """Post the output of the analysis in the means appropriate for the CI environment.""" - # This is a bit of a placeholder for now. The output works in that it is human readable. - # However, it is more meant for display on the web, as HTML and rendered Markdown. - print(f" [+] Analysis output:\n{self.analysis_output}") + print(f" [+] Analysis output:\n{render(self.analysis_output)}") diff --git a/src/phylum/ci/ci_precommit.py b/src/phylum/ci/ci_precommit.py index db260c05..016a0f17 100644 --- a/src/phylum/ci/ci_precommit.py +++ b/src/phylum/ci/ci_precommit.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import List, Optional -from phylum.ci import SCRIPT_NAME +from connect.utils.terminal.markdown import render from phylum.ci.ci_base import CIBase @@ -35,9 +35,26 @@ def _check_prerequisites(self) -> None: cmd = "git diff --cached --name-only".split() staged_files = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip().split("\n") + extra_arg_paths = (Path(extra_arg).resolve() for extra_arg in self.extra_args) + + print(" [*] Checking extra args for valid pre-commit scenarios ...") + # Allow for a pre-commit config set up to send all staged files to the hook if sorted(staged_files) == sorted(self.extra_args): print(" [+] The extra args provided exactly match the list of staged files") + # Allow for a pre-commit config set up to filter the files sent to the hook + elif all(extra_arg in staged_files for extra_arg in self.extra_args): + print(" [+] All the extra args are staged files") + # Allow for cases where the lockfile is included or explicitly specified (e.g., `pre-commit run --all-files`) + elif self.lockfile in extra_arg_paths: + print(" [+] The lockfile was included in the extra args") + # NOTE: There is still the case where the lockfile is "accidentally" included as an extra argument. For example, + # `phylum-ci poetry.lock` was used instead of `phylum-ci --lockfile poetry.lock`, which is bad syntax but + # nonetheless results in the `CIPreCommit` environment used instead of `CINone`. This is not terrible; it + # just might be a slightly confusing corner case. It might be possible to use a library like `psutil` to + # acquire the command line from the parent process and inspect it for `pre-commit` usage. That is a + # heavyweight solution and one that will not be pursued until the need for it is more clear. else: + print(" [+] No valid pre-commit scenario found. Bailing ...") raise SystemExit(f" [!] Unrecognized arguments: {' '.join(self.extra_args)}") @property @@ -50,7 +67,7 @@ def phylum_label(self) -> str: # Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects cmd = f"git hash-object {self.lockfile}".split() lockfile_hash_object = subprocess.run(cmd, check=True, text=True, capture_output=True).stdout.strip() - label = f"{SCRIPT_NAME}_{self.ci_platform_name}_{current_branch}_{lockfile_hash_object}" + label = f"{self.ci_platform_name}_{current_branch}_{lockfile_hash_object[:7]}" label = label.replace(" ", "-") return label @@ -77,6 +94,4 @@ def _is_lockfile_changed(self, lockfile: Path) -> bool: def post_output(self) -> None: """Post the output of the analysis in the means appropriate for the CI environment.""" - # TODO: Change this placeholder when the real Python pre-commit hook is ready. - # https://github.com/phylum-dev/phylum-ci/issues/35 - print(f" [+] Analysis output:\n{self.analysis_output}") + print(f" [+] Analysis output:\n{render(self.analysis_output)}") diff --git a/src/phylum/ci/constants.py b/src/phylum/ci/constants.py index 944c2833..fa6b78f3 100644 --- a/src/phylum/ci/constants.py +++ b/src/phylum/ci/constants.py @@ -5,7 +5,7 @@ from phylum.ci.common import RiskDomain # The common Phylum header that must exist as the first text in the first line of all analysis output -PHYLUM_HEADER = "## Phylum OSS Supply Chain Risk Analysis" +PHYLUM_HEADER = "# Phylum OSS Supply Chain Risk Analysis" # NOTE: All multi-line strings are indented by two levels on purpose, to ensure they line up correctly when used with # each other in templates and are all fully left justified after applying `textwrap.dedent` for normalization.