From 533e05114e9b8766abe3be3d22a0f864342a013b Mon Sep 17 00:00:00 2001 From: grafanabot Date: Thu, 28 Nov 2024 07:11:11 +0000 Subject: [PATCH] Update `make docs` procedure --- docs/docs.mk | 31 +-- docs/make-docs | 526 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 419 insertions(+), 138 deletions(-) diff --git a/docs/docs.mk b/docs/docs.mk index c4ba766fb..c0aae10ba 100644 --- a/docs/docs.mk +++ b/docs/docs.mk @@ -34,11 +34,6 @@ endif # First project is considered the primary one used for doc-validator. PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS))))) -# Name for the container. -ifeq ($(origin DOCS_CONTAINER), undefined) -export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs -endif - # Host port to publish container port to. ifeq ($(origin DOCS_HOST_PORT), undefined) export DOCS_HOST_PORT := 3002 @@ -70,6 +65,11 @@ ifeq ($(origin HUGO_REFLINKSERRORLEVEL), undefined) export HUGO_REFLINKSERRORLEVEL := WARNING endif +# Whether to pull the latest container image before running the container. +ifeq ($(origin PULL), undefined) +export PULL := true +endif + .PHONY: docs-rm docs-rm: ## Remove the docs container. $(PODMAN) rm -f $(DOCS_CONTAINER) @@ -86,13 +86,12 @@ make-docs: fi .PHONY: docs -docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. See also `docs-no-pull`. +docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. To not pull the image, set `PULL=false`. +ifeq ($(PULL), true) docs: docs-pull make-docs - $(CURDIR)/make-docs $(PROJECTS) - -.PHONY: docs-no-pull -docs-no-pull: ## Serve documentation locally without pulling the `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. -docs-no-pull: make-docs +else +docs: make-docs +endif $(CURDIR)/make-docs $(PROJECTS) .PHONY: docs-debug @@ -101,13 +100,19 @@ docs-debug: make-docs WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(CURDIR)/make-docs $(PROJECTS) .PHONY: doc-validator -doc-validator: ## Run doc-validator on the entire docs folder. +doc-validator: ## Run doc-validator on the entire docs folder which includes pulling the latest `DOC_VALIDATOR_IMAGE` (default: `grafana/doc-validator:latest`) container image. To not pull the image, set `PULL=false`. doc-validator: make-docs +ifeq ($(PULL), true) + $(PODMAN) pull -q $(DOC_VALIDATOR_IMAGE) +endif DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(CURDIR)/make-docs $(PROJECTS) .PHONY: vale -vale: ## Run vale on the entire docs folder. +vale: ## Run vale on the entire docs folder which includes pulling the latest `VALE_IMAGE` (default: `grafana/vale:latest`) container image. To not pull the image, set `PULL=false`. vale: make-docs +ifeq ($(PULL), true) + $(PODMAN) pull -q $(VALE_IMAGE) +endif DOCS_IMAGE=$(VALE_IMAGE) $(CURDIR)/make-docs $(PROJECTS) .PHONY: update diff --git a/docs/make-docs b/docs/make-docs index 97dcbf5b0..2dc6726d9 100755 --- a/docs/make-docs +++ b/docs/make-docs @@ -5,103 +5,246 @@ # Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/. # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes. # Changes are relevant to this script and the support docs.mk GNU Make interface. +# +# ## 8.1.0 (2024-08-22) +# +# ### Added +# +# - Additional website mounts for projects that use the website repository. +# +# Mounts are required for `make docs` to work in the website repository or with the website project. +# The Makefile is also mounted for convenient development of the procedure that repository. +# +# ## 8.0.1 (2024-07-01) +# +# ### Fixed +# +# - Update log suppression to catch new format of website /docs/ homepage REF_NOT_FOUND warnings. +# +# These warnings are related to missing some pages during the build that are required for the /docs/ homepage. +# They were previously suppressed but the log format changed and without this change they reappear in the latest builds. +# +# ## 8.0.0 (2024-05-28) +# +# ### Changed +# +# - Add environment variable `OUTPUT_FORMAT` to control the output of commands. +# +# The default value is `human` and means the output format is human readable. +# The value `json` is also supported and outputs JSON. +# +# Note that the `json` format isn't supported by `make docs`, only `make doc-validator` and `make vale`. +# +# ## 7.0.0 (2024-05-03) +# +# ### Changed +# +# - Pull images for all recipes that use containers by default. +# +# Use the `PULL=false` variable to disable this behavior. +# +# ### Removed +# +# - The `docs-no-pull` target as it's redundant with the new `PULL=false` variable. +# +# ## 6.1.0 (2024-04-22) +# +# ### Changed +# +# - Mount volumes with SELinux labels. +# +# https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label +# +# ### Added +# +# - Pseudo project for including only website resources and no website content. +# +# Facilitates testing shortcodes and layout changes with a small documentation set instead of Grafana Cloud or the entire website. +# +# ## 6.0.1 (2024-02-28) +# +# ### Added +# +# - Suppress new errors relating to absent content introduced in https://github.com/grafana/website/pull/17561. +# +# ## 6.0.0 (2024-02-16) +# +# ### Changed +# +# - Require `jq` for human readable `make doc-validator` output. +# +# ## 5.4.0 (2024-02-12) +# +# ### Changed +# +# - Set `WEBSITE_MOUNTS=true` when a user includes the `website` project. +# +# Ensures consistent behavior across repositories. +# To disable website mounts, add `export WEBSITE_MOUNTS := false` to your `variables.mk` or `variables.mk.local` file. +# - Use website mounts and container volumes also when a user includes the `grafana-cloud` project. +# +# ## 5.3.0 (2024-02-08) +# +# ### Changed +# +# - Updated support for plugins monorepo now that multiple projects have been moved into it. +# - Use `printf` instead of `echo` for better portability of output. +# +# https://www.in-ulm.de/~mascheck/various/echo+printf/ +# +# ## 5.2.0 (2024-01-18) +# +# ### Changed +# +# - Updated `make vale` to use latest Vale style and configuration. +# - Updated `make vale` to use platform appropriate image. +# +# ## 5.1.2 (2023-11-08) +# +# ### Added +# +# - Hide manual_mount warning messages from non-debug output. +# Set the DEBUG environment variable to see all hidden messages. +# +# ## 5.1.1 (2023-10-30) +# +# ### Added +# +# - Support for Datadog and Oracle data source plugins repositories. +# +# ## 5.1.0 (2023-10-20) +# +# ### Added +# +# - Support for the plugins monorepo. +# +# ## 5.0.0 (2023-10-18) +# +# ### Added +# +# - Improved support for website repository. +# +# Mount more content and provide some feedback to users that the build can take time. +# +# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable. +# +# ### Fixed +# +# - Correct key combination for interrupting the process. +# +# Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press. +# +# ### Removed +# +# - Imperfect implementation of container name. +# +# Facilitates running `make vale` and `make docs` at once. +# Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides. +# +# - Forced platform specification now that multiple architecture images exist. +# +# Significantly speeds up build times on larger repositories. +# +# ## 4.2.2 (2023-10-05) +# - Added support for Jira data source and MongoDB data source plugins repositories. +# # ## 4.2.1 (2023-09-13) # ## Fixed - +# # - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port. - +# # ## 4.2.0 (2023-09-01) - +# # ### Added - +# # - Retry the initial webserver request up to ten times to allow for the process to start. # If it is still failing after ten seconds, an error message is logged. - +# # ## 4.1.1 (2023-07-20) - +# # ### Fixed - +# # - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH. - +# # ## 4.1.0 (2023-06-16) - +# # ### Added - +# # - Mounts of `layouts` and `config` directories for the `website` project. # Ensures that local changes to mounts or shortcodes are reflected in the development server. - +# # ### Fixed - +# # - Version inference for versioned docs pages. # Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds. - +# # ## 4.0.0 (2023-06-06) - +# # ### Removed - +# # - `doc-validator/%` target. # The behavior of the target was not as described. # Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files. - +# # ## 3.0.0 (2023-05-18) - +# # ### Fixed - +# # - Compatibility with the updated Make targets in the `website` repository. # `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment. - +# # ## 2.0.0 (2023-05-18) - +# # ### Added - +# # - Support for the grafana-cloud/frontend-observability/faro-web-sdk project. # - Use of `doc-validator` v2.0.x which includes breaking changes to command line options. - +# # ### Fixed - +# # - Source grafana-cloud project from website repository. - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ## 1.2.1 (2023-05-05) - +# # ### Fixed - +# # - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version. # - Fix mounting multiple projects broken by the changes in 1.0.1 - +# # ## 1.2.0 (2023-05-05) - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ### Fixed - +# # ## 1.1.0 (2023-05-05) - +# # ### Added - +# # - Rewrite error output so it can be followed by text editors. - +# # ### Fixed - +# # - Fix `docs-debug` container process port. - +# # ## 1.0.1 (2023-05-04) - +# # ### Fixed - +# # - Ensure complete section hierarchy so that all projects have a visible menu. - +# # ## 1.0.0 (2023-05-04) - +# # ### Added - +# # - Build multiple projects simultaneously if all projects are checked out locally. # - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects. # - Redirect project root to mounted version. @@ -131,7 +274,6 @@ set -ef -readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}" readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}" readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}" @@ -141,8 +283,8 @@ readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}" readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}" readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}" readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}" -# If set, the docs-base image will run a prebuild script that sets up Hugo mounts. -readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}" + +readonly OUTPUT_FORMAT="${OUTPUT_FORMAT:-human}" PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)" @@ -213,7 +355,9 @@ SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk' SOURCES_helm_charts_mimir_distributed='mimir' SOURCES_helm_charts_tempo_distributed='tempo' SOURCES_opentelemetry='opentelemetry-docs' -SOURCES_plugins_grafana_splunk_datasource='splunk-datasource' +SOURCES_plugins_grafana_datadog_datasource='datadog-datasource' +SOURCES_plugins_grafana_oracle_datasource='oracle-datasource' +SOURCES_resources='website' VERSIONS_as_code='UNVERSIONED' VERSIONS_grafana_cloud='UNVERSIONED' @@ -223,6 +367,9 @@ VERSIONS_grafana_cloud_k6='UNVERSIONED' VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED' VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED' VERSIONS_opentelemetry='UNVERSIONED' +VERSIONS_plugins_grafana_datadog_datasource='latest' +VERSIONS_plugins_grafana_oracle_datasource='latest' +VERSIONS_resources='UNVERSIONED' VERSIONS_technical_documentation='UNVERSIONED' VERSIONS_website='UNVERSIONED' VERSIONS_writers_toolkit='UNVERSIONED' @@ -231,8 +378,11 @@ PATHS_grafana_cloud='content/docs/grafana-cloud' PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed' PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed' PATHS_mimir='docs/sources/mimir' +PATHS_plugins_grafana_datadog_datasource='docs/sources' +PATHS_plugins_grafana_oracle_datasource='docs/sources' +PATHS_resources='content' PATHS_tempo='docs/sources/tempo' -PATHS_website='content/docs' +PATHS_website='content' # identifier STR # Replace characters that are not valid in an identifier with underscores. @@ -247,6 +397,77 @@ aget() { eval echo '$'"$(identifier "$1")_$(identifier "$2")" } +# src returns the project source repository name for a project. +src() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo plugins-private + else + aget SOURCES "${_project}" + fi + ;; + *) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo "${_project}" + else + aget SOURCES "${_project}" + fi + ;; + esac + + unset _project +} + +# path returns the relative path within the repository that contain the docs for a project. +path() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "${_project}/docs/sources" + else + aget PATHS "${_project}" + fi + ;; + *) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "docs/sources" + else + aget PATHS "${_project}" + fi + esac + + unset _project +} + +# version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'. +version() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo "UNVERSIONED" + else + aget VERSIONS "${_project}" + fi + ;; + *) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo latest + else + aget VERSIONS "${_project}" + fi + esac + + unset _project +} + + # new_proj populates a new project structure. new_proj() { _project="$1" @@ -257,31 +478,19 @@ new_proj() { # If version is not set, use the script mapping of project to default versions if it exists. # Fallback to 'latest'. if [ -z "${_version}" ]; then - if [ -z "$(aget VERSIONS "${_project}")" ]; then - _version=latest - else - _version="$(aget VERSIONS "${_project}")" - fi + _version="$(version "${_project}")" fi # If repo is not set, use the script mapping of project to repo name if it exists. # Fallback to using the project name. if [ -z "${_repo}" ]; then - if [ -z "$(aget SOURCES "${_project}")" ]; then - _repo="${_project}" - else - _repo="$(aget SOURCES "${_project}")" - fi + _repo="$(src "${_project}")" fi # If path is not set, use the script mapping of project to docs sources path if it exists. # Fallback to using 'docs/sources'. if [ -z "${_path}" ]; then - if [ -z "$(aget PATHS "${_project}")" ]; then - _path="docs/sources" - else - _path="$(aget PATHS "${_project}")" - fi + _path="$(path "${_project}")" fi echo "${_project}:${_version}:${_repo}:${_path}" @@ -295,7 +504,7 @@ proj_url() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then + if [ "${_project}" = website ]; then echo "http://localhost:${DOCS_HOST_PORT}/docs/" unset _project _version @@ -329,8 +538,8 @@ proj_dst() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then - echo '/hugo/content/docs' + if [ "${_project}" = website ]; then + echo '/hugo/content' unset _project _version return @@ -388,7 +597,7 @@ proj_canonical() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then + if [ "${_project}" = website ]; then echo '/docs' unset _project _version @@ -435,6 +644,11 @@ POSIX_HERESTRING proj_to_url_src_dst_ver "$(new_proj helm-charts/mimir-distributed "${_version}")" proj_to_url_src_dst_ver "$(new_proj enterprise-metrics "${_version}")" ;; + resources) + _repo="$(repo_path website)" + echo "arbitrary^${_repo}/config^/hugo/config" "arbitrary^${_repo}/layouts^/hugo/layouts" "arbitrary^${_repo}/scripts^/hugo/scripts" + unset _repo + ;; traces) proj_to_url_src_dst_ver "$(new_proj tempo "${_version}")" proj_to_url_src_dst_ver "$(new_proj enterprise-traces "${_version}")" @@ -457,50 +671,48 @@ await_build() { while [ "${i}" -ne "${max}" ] do sleep 1 - debg "Retrying request to webserver assuming the process is still starting up." + debg "Retrying request to web server assuming the process is still starting up." i=$((i + 1)) if ${req} "${url}"; then - echo - echo "View documentation locally:" + printf '\r\nView documentation locally:\r\n' for x in ${url_src_dst_vers}; do IFS='^' read -r url _ _ <&2 + printf 'DEBG: %s\r\n' "$1" >&2 fi } errr() { - echo "ERRR: $1" >&2 + printf 'ERRR: %s\r\n' "$1" >&2 } note() { - echo "NOTE: $1" >&2 + printf 'NOTE: %s\r\n' "$1" >&2 } url_src_dst_vers="$(url_src_dst_vers "$@")" @@ -512,11 +724,23 @@ for arg in "$@"; do IFS=: read -r _project _ _repo _ </dev/null 2>&1; then + errr '`jq` must be installed for the `doc-validator` target to work.' + note 'To install `jq`, refer to https://jqlang.github.io/jq/download/,' + + exit 1 + fi + + ${cmd} \ + | jq -r '"ERROR: \(.location.path):\(.location.range.start.line // 1):\(.location.range.start.column // 1): \(.message)" + if .suggestions[0].text then "\nSuggestion: \(.suggestions[0].text)" else "" end' + ;; + json) + ${cmd} + ;; + *) # default + errr "Invalid output format '${OUTPUT_FORMAT}'" + esac ;; 'grafana/vale') proj="$(new_proj "$1")" - echo - "${PODMAN}" run \ + printf '\r\n' + IFS='' read -r cmd <"${tempfile}" #!/usr/bin/env bash + +tc() { + set \${*,,} + echo \${*^} +} + for redirect in ${redirects}; do IFS='^' read -r path ver <<<"\${redirect}" echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}" @@ -603,8 +869,12 @@ done for x in "${url_src_dst_vers}"; do IFS='^' read -r _ _ dst _ <<<"\${x}" + title="\${dst%/*}" + title="\$(tc \${title##*/})" while [[ -n "\${dst}" ]]; do - touch "\${dst}/_index.md" + if [[ ! -f "\${dst}/_index.md" ]]; then + echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md" + fi dst="\${dst%/*}" done done @@ -616,7 +886,7 @@ fi ${WEBSITE_EXEC} EOF chmod +x "${tempfile}" - volumes="${volumes} --volume=${tempfile}:/entrypoint" + volumes="${volumes} --volume=${tempfile}:/entrypoint:z" readonly volumes IFS='' read -r cmd <&1| sed \ - -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ - -e '/^hugo server/ d' \ - -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ - -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ - -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ - -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ - -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ - -e '/website-proxy/ d' \ - -e '/rm -rf dist*/ d' \ - -e '/Press Ctrl+C to stop/ d' \ - -e '/make/ d' || echo + await_build http://localhost:3003 & + + ${cmd} /entrypoint 2>&1\ + | sed -u \ + -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ + -e '/^hugo server/ d' \ + -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ + -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ + -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ + -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ + -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ + -e '/website-proxy/ d' \ + -e '/rm -rf dist*/ d' \ + -e '/Press Ctrl+C to stop/ d' \ + -e '/make/ d' \ + -e '/WARNING: The manual_mount source directory/ d' \ + -e '/"docs\/_index.md" not found/d' fi ;; esac