docs: Generate documentation with sphinx

.github/workflows/main.yml

@@ -62,6 +62,45 @@ jobs:
       run: |
         HYPOTHESIS_PROFILE=ci make test
 
+  docs:
+    runs-on: ubuntu-latest
+    needs: [cache]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        buildkitd-flags: --debug
+
+    - name: Build container
+      uses: docker/build-push-action@v5
+      with:
+        context: compose
+        push: false
+        load: true
+        tags: ghcr.io/adfinis/pyaptly/cache:latest
+        cache-from: type=registry,ref=ghcr.io/adfinis/pyaptly/cache:gha
+
+    - name: Generate the docs
+      run: |
+        make docs
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: html-docs
+        path: docs/_build/html/
+
+    - name: Deploy
+      uses: peaceiris/actions-gh-pages@v4
+      if: github.ref == 'refs/heads/main'
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: docs/_build/html
+
   package:
     runs-on: ubuntu-latest
     needs: [test]

.github/workflows/pull-requests.yml

@@ -31,3 +31,34 @@ jobs:
     - name: Run tests
       run: |
         HYPOTHESIS_PROFILE=ci make test
+
+  docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        buildkitd-flags: --debug
+
+    - name: Build container
+      uses: docker/build-push-action@v5
+      with:
+        context: compose
+        push: false
+        load: true
+        tags: ghcr.io/adfinis/pyaptly/cache:latest
+        cache-from: type=registry,ref=ghcr.io/adfinis/pyaptly/cache:gha
+
+    - name: Generate the docs
+      run: |
+        make docs
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: html-docs
+        path: docs/_build/html/

.gitignore

@@ -3,3 +3,8 @@ __pycache__
 /.dmypy.json
 /.coverage
 /dist
+docs/cli
+docs/sphinx-template
+docs/_build
+docs/Command_Line_Reference.md
+docs/Config_Reference.md

Makefile

@@ -63,6 +63,31 @@ lint-code:  ## check all linters
 .PHONY: test
 test: pytest mypy lint-code ## run all testing
 
+
+.PHONY: docs
+docs: poetry-install ## generate documentation
+	rm -vrf docs/_build/* docs/cli/*
+	[[ -d docs/sphinx-template ]] || git clone https://github.com/adfinis-sygroup/adsy-sphinx-template docs/sphinx-template
+	@docker compose exec testing bash -c "poetry install --only docs"
+
+	# Currently md-click has a bad dependency on old click. We fix this with the next command
+	@docker compose exec testing bash -c "poetry run pip install md-click==1.0.1"
+	@docker compose exec testing bash -c "patch -f /root/.cache/pypoetry/virtualenvs/pyaptly-*-py3.11/lib/python3.11/site-packages/md_click/main.py md-click.patch || true"
+
+	# Generate CLI & Config docs
+	@docker compose exec testing bash -c "mkdir -p docs/_temp && poetry run mdclick dumps --baseModule=pyaptly.cli --baseCommand=pyaptly --docsPath=./docs/_temp"
+	@docker compose exec testing bash -c "poetry run jsonschema2md pyaptly/config.schema.json docs/Config_Reference.md"
+
+	@cat docs/_temp/pyaptly.md docs/_temp/pyaptly-*.md > docs/Command_Line_Reference.md
+	# mdclick has not enough indentation
+	@sed -i 's/^#/##/' docs/Command_Line_Reference.md
+	# Add title, must happen after the above sed command
+	@sed -i '1a<!-- This file is generated with mdclick -->\n# Command Line Reference' docs/Command_Line_Reference.md
+	@docker compose exec testing bash -c "rm -r docs/_temp"
+
+	# SPHINX Render
+	@docker compose exec testing bash -c 'cd docs/ && poetry run make html'
+
 .PHONY: shell
 shell: poetry-install ## run shell
 	@docker compose exec testing bash -c "SHELL=bash poetry shell"

README.md

@@ -8,7 +8,7 @@ input files.
 - For for the old version [switch to the master branch](https://github.com/adfinis/pyaptly/tree/master)
 - Main branch builds contain [alpha packages](https://github.com/adfinis/pyaptly/actions/runs/8147002919), see Artifacts
 
-# Why & How
+## Why & How
 
 [Aptly](https://www.aptly.info/) is great tool for creating Debian repositories.
 But as soon as it's required to maintain repositories for different [environments](https://en.wikipedia.org/wiki/Deployment_environment) it gets very complicated fast.
@@ -22,8 +22,6 @@ That means it's hard to roll back to a previous state if required.
 This problem is fixed by using fix timestamps in snapshot names.
 That behaviour also allows to define a fixed update spacing. It's possible to say for example "only update this snapshot once a week".
 
-[Follow the Tutorial](./docs/TUTORIAL.md)
-
 ## Example commands
 
 Initialize a new aptly server.

docs/Code_Reference.rst

@@ -0,0 +1,41 @@
+Code Reference
+==============
+
+The following code reference is extracted from the source code and mainly used for debugging/development.
+Users of `pyaptly` can ignore this.
+
+Main
+----
+.. automodule:: pyaptly.main
+   :members:
+
+Mirror
+------
+.. automodule:: pyaptly.mirror
+   :members:
+
+Repo
+----
+.. automodule:: pyaptly.repo
+   :members:
+
+Snapshot
+--------
+.. automodule:: pyaptly.snapshot
+   :members:
+
+Publish
+-------
+.. automodule:: pyaptly.publish
+   :members:
+
+State Reader
+------------
+.. automodule:: pyaptly.state_reader
+   :members:
+
+Util
+----
+.. automodule:: pyaptly.util
+   :members:
+

docs/TUTORIAL.md → docs/Getting_Started.md

@@ -1,3 +1,5 @@
+# Getting Started
+
 > Note: This tutorial assumes basic knowledge of [Aptly](https://www.aptly.info/).
 
 Pyaptly is capable of managing mirrors, snapshots and publishes.
@@ -6,11 +8,11 @@ But for the purpose of this tutorial we assume a clean [install of Aptly](https:
 
 TODO: Note to jump to the relevant chapter if only a subset should be managed by aptly.
 
-# Installation
+## Installation
 
 TODO (once packages are available)
 
-# Aptly Mirror
+## Aptly Mirror
 
 Pyaptly can create and update mirrors. Since mirrors are nor a very complicated construct, there's no extra logic not available within aptly.
 Configuring a mirror with pyaptly is pretty much the same as writing a command for aptly - except that it's declarative.
@@ -42,7 +44,7 @@ Pyaptly also takes care of downloading the gpp key if it isn't availble yet. If
 
 > For a list of all configuration options of a mirror, check out [the reference](TODO: Reference link).
 
-## updating mirrors
+### updating mirrors
 
 We can also tell pyaptly to update all defined mirrors:
 ```bash
@@ -52,9 +54,9 @@ pyaptly mirror ./config.toml update
 This is exactly the same as `aptly mirror update aptly` with the above config.
 But it will update all defined mirrors if more than one is defined, making it a bit more convenient than using `aptly` directly.
 
-# Snapshots
+## Snapshots
 
-## Basic snapshots
+### Basic snapshots
 
 Pyaptly has some extra features for snapshots, but let's start by creating a very simple snapshot first.
 
@@ -87,7 +89,7 @@ As you see, `pyaptly` first "rotates" the snapshot by just renaming and postfixi
 
 > Similar to mirrors, pyaptly allows a variety of configuration options for snapshots. Check out [the reference](TODO: Link to reference).
 
-## Snapshots with retention
+### Snapshots with retention
 
 Snapshots with retention are a bit more complicated than simple snapshots.
 The retention time is either 1 day or 1 week. Other types of retention are currently not implemented.
@@ -134,7 +136,7 @@ It's also important to understand that `pyaptly snapshot config.toml update` wil
 If we were to patch our systems only once a week, then what we want is to uncomment the line `repeat-weekly: "mon"`. This way, our snapshot would be backdated a full day to `aptly-20240102T0000Z`.
 This means that pyaptly would only create a new snapshot once a week, no matter how often the command has run.
 
-# Publish
+## Publish
 
 Pyaptly publishes also come with some extra sugar building on the features of the snapshots. But let's start with a simple publish again:
 ```bash

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.md

@@ -0,0 +1 @@
+../README.md

docs/conf.py

@@ -0,0 +1,48 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Pyaptly'
+copyright = 'Adfinis AG'
+author = 'Adfinis AG'
+release = '2.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    'myst_parser',
+    'sphinx.ext.autosectionlabel',
+    'sphinx.ext.autodoc',
+]
+myst_enable_extensions = [
+    "linkify",
+    "replacements",
+    "smartquotes",
+]
+
+# For autodoc
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path('..').resolve()))
+
+autosectionlabel_prefix_document = True
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinx-template']
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'adsy'
+html_theme_path = ['sphinx-template/html']
+html_title = project
+html_static_path = ['sphinx-template/html/adsy/static/']
+
+# Output file base name for HTML help builder.
+#htmlhelp_basename = 'test1234'
+

docs/index.rst

@@ -0,0 +1,20 @@
+.. Pyaptly documentation master file, created by
+   sphinx-quickstart on Thu Nov  7 16:17:24 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+.. include:: README.md
+   :parser: myst_parser.sphinx_
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Contents:
+
+   README.md
+   Getting_Started.md
+   Config_Reference.md
+   Command_Line_Reference.md
+   Code_Reference.rst
+
+.. automodule:: pyaptly
+   :members:

md-click.patch

@@ -0,0 +1,20 @@
++++ ./a ./b	2024-11-08 13:56:06.904479048 +0000
+@@ -49,12 +49,12 @@
+         parent = helpdct.get("parent", "") or ''
+         options = {
+             opt.name: {
+-                "usage": '\n'.join(opt.opts),
+-                "prompt": opt.prompt,
+-                "required": opt.required,
+-                "default": opt.default,
+-                "help": opt.help,
+-                "type": str(opt.type)
++                "usage": '\n'.join(opt.opts),
++                "prompt": getattr(opt, "prompt", None),
++                "required": getattr(opt, "required", None),
++                "default": getattr(opt, "default", None),
++                "help": getattr(opt, "help", None),
++                "type": str(getattr(opt, "type", None))
+             }
+             for opt in helpdct.get('params', [])
+         }