Merge branch 'master' into ez_patch

.github/workflows/publish_doc.yml

@@ -38,19 +38,19 @@ jobs:
         with:
           path: ~/.cache/pip
           key: pip-${{ hashFiles('**/setup.py') }}
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
           architecture: "x64"
       - name: check OS
         run: cat /etc/os-release
       - name: install dependencies
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc
+          sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc nodejs npm
       - name: install espnet
         env:
-          ESPNET_PYTHON_VERSION: 3.8
+          ESPNET_PYTHON_VERSION: 3.10
           TH_VERSION: 2.0.1
           CHAINER_VERSION: 6.0.0
           USE_CONDA: false
@@ -62,4 +62,4 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: doc/build
+          publish_dir: ./dist

ci/doc.sh

@@ -1,38 +1,116 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 . tools/activate_python.sh
 
+clean_outputs() {
+    rm -rf dist
+    rm -rf espnet_bin
+    rm -rf espnet2_bin
+    rm -rf utils_py
+
+    rm -rf doc/_gen
+    rm -rf doc/build
+
+    rm -rf doc/vuepress/src/*.md
+    rm -rf doc/vuepress/src/notebook
+    rm -rf doc/vuepress/src/*
+    rm -rf doc/vuepress/src/.vuepress/.temp
+    rm -rf doc/vuepress/src/.vuepress/.cache
+}
+
+build_and_convert () {
+    # $1: path
+    # $2: output
+    mkdir -p ./doc/_gen/tools/$2
+    for filename in $1; do
+        bn=$(basename ${filename})
+        echo "Converting ${filename} to rst..."
+        ./doc/usage2rst.sh ${filename} > ./doc/_gen/tools/$2/${bn}.rst
+    done
+}
+
 if [ ! -e tools/kaldi ]; then
     git clone https://github.com/kaldi-asr/kaldi --depth 1 tools/kaldi
 fi
 
+# clean previous build
+clean_outputs
+
 # build sphinx document under doc/
 mkdir -p doc/_gen
+mkdir -p doc/_gen/tools
+mkdir -p doc/_gen/guide
 
 # NOTE allow unbound variable (-u) inside kaldi scripts
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH-}
 set -euo pipefail
 # generate tools doc
-(
-    cd ./utils
-    ../doc/argparse2rst.py ./*.py > ../doc/_gen/utils_py.rst
-)
+mkdir utils_py
+./doc/argparse2rst.py \
+    --title utils_py \
+    --output_dir utils_py \
+    ./utils/*.py
+mv utils_py ./doc/_gen/tools
 
-./doc/argparse2rst.py ./espnet/bin/*.py > ./doc/_gen/espnet_bin.rst
-# FIXME
-# ./doc/argparse2rst.py ./espnet2/bin/*.py > ./doc/_gen/espnet2_bin.rst
+mkdir espnet_bin
+./doc/argparse2rst.py \
+    --title espnet_bin \
+    --output_dir espnet_bin \
+    ./espnet/bin/*.py
+mv espnet_bin ./doc/_gen/tools
 
+mkdir espnet2_bin
+./doc/argparse2rst.py \
+    --title espnet2_bin \
+    --output_dir espnet2_bin \
+    ./espnet2/bin/*.py
+mv espnet2_bin ./doc/_gen/tools
 
-find ./utils/*.sh tools/sentencepiece_commands/spm_* -exec ./doc/usage2rst.sh {} \; | tee ./doc/_gen/utils_sh.rst
-find ./espnet2/bin/*.py -exec ./doc/usage2rst.sh {} \; | tee ./doc/_gen/espnet2_bin.rst
+build_and_convert "utils/*.sh" utils
+build_and_convert "tools/sentencepiece_commands/spm_decode" spm
+build_and_convert "tools/sentencepiece_commands/spm_encode" spm
+# There seems no help prepared for spm_train command.
 
-./doc/notebook2rst.sh > ./doc/_gen/notebooks.rst
+./doc/notebook2rst.sh > ./doc/notebooks.md
 
 # generate package doc
-./doc/module2rst.py --root espnet espnet2 --dst ./doc --exclude espnet.bin
+python ./doc/members2rst.py --root espnet --dst ./doc/_gen/guide --exclude espnet.bin
+python ./doc/members2rst.py --root espnet2 --dst ./doc/_gen/guide --exclude espnet2.bin
+python ./doc/members2rst.py --root espnetez --dst ./doc/_gen/guide
+
+# build markdown
+cp ./doc/index.rst ./doc/_gen/index.rst
+cp ./doc/conf.py ./doc/_gen/
+rm -f ./doc/_gen/tools/espnet2_bin/*_train.rst
+sphinx-build -M markdown ./doc/_gen ./doc/build
+
+# copy markdown files to specific directory.
+cp -r ./doc/build/markdown/* ./doc/vuepress/src/
+cp -r ./doc/notebook ./doc/vuepress/src/
+cp ./doc/*.md ./doc/vuepress/src/
+mv ./doc/vuepress/src/README.md ./doc/vuepress/src/document.md
+cp -r ./doc/image ./doc/vuepress/src/
+
+# Document generation has finished.
+# From the following point we modify files for VuePress.
+# replace language tags which is not supported by VuePress
+# And convert custom tags to &lt; and &gt;, as <custom tag> can be recognized a html tag.
+python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/guide
+python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/tools
+
+# Convert API document to specific html tags to display sphinx style
+python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/guide/
+python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/tools/
+
+# Create navbar and sidebar.
+cd ./doc/vuepress
+python create_menu.py --root ./src
 
-# build html
-# TODO(karita): add -W to turn warnings into errors
-sphinx-build -b html doc doc/build
+npm i
+# npm run docs:dev
+npm run docs:build
+mv src/.vuepress/dist ../../
 
-touch doc/build/.nojekyll
+touch ../../dist/.nojekyll

doc/README.md

@@ -5,11 +5,30 @@
 We use [sphinx](https://www.sphinx-doc.org) to generate HTML documentation.
 
 ```sh
+# Clean conda env for docs
 $ cd <espnet_root>
+$ conda create -p ./envs python=3.10
+$ conda activate ./envs
+
+# Requirements
+$ pip install -e ".[all]"
 $ pip install -e ".[doc]"
+$ conda install conda-forge::ffmpeg
+$ conda install conda-forge::nodejs==22.6.0
+
+# (Optional requirement) To use flake8-docstrings
 $ pip install -U flake8-docstrings
 ```
 
+If you used the above clean conda environment, you have write your own `. tools/activate_python.sh`.
+The example will be:
+```sh
+#!/usr/bin/env bash
+
+# You might check $CONDA_EXE to find the <conda_root>
+. <conda_root>/miniconda/etc/profile.d/conda.sh && conda activate <espnet_root>/envs
+```
+
 ## Style check using flake8-docstrings
 
 You can check that your docstring style is correct by `ci/test_flake8.sh` using [flake8-docstrings](https://pypi.org/project/flake8-docstrings/).
@@ -39,15 +58,13 @@ DO NOT ADD NEW FILES TO THIS BLACK LIST!
 
 ## Generate HTML
 
-You can generate local HTML manually using sphinx Makefile
-
+You can generate and test the webpage using sphinx Makefile.
 ```sh
 $ cd <espnet_root>
 $ ./ci/doc.sh
+$ npm run docs:dev
 ```
 
-open `doc/build/index.html`
-
 ## Deploy
 
 When your PR is merged into `master` branch, our [CI](https://github.com/espnet/espnet/blob/master/.github/workflows/doc.yml) will automatically deploy your sphinx html into https://espnet.github.io/espnet/.

doc/about_this_doc.md

@@ -0,0 +1,66 @@
+
+# Developer's Guide to the ESPnet Homepage
+
+This document outlines the process of automatically generating the ESPnet homepage. It provides step-by-step instructions for building the homepage and details the underlying operations during the generation process.
+
+## Building the Homepage
+
+1. **Clone the ESPnet Repository**: Begin by cloning the ESPnet repository from GitHub.
+
+2. **Generate the `activate_python.sh` Script**:
+   - You can generate this file by running either `setup_anaconda.sh` or `setup_venv.sh`.
+   - Alternatively, create your own virtual environment and manually write the command to activate it in `activate_python.sh`.
+
+3. **Run `activate_python.sh`**: Execute this script to activate the Python environment.
+
+4. **Install the Dependencies**:
+   Install the necessary dependencies using the following commands:
+   ```
+   espnet[all]
+   espnet[doc]
+   k2
+   chainer
+   ```
+
+5. **Build the Homepage**:
+   Run the following script to generate the homepage:
+   ```
+   ./ci/doc.sh
+   ```
+
+## Key Points
+
+- The homepage is built using VuePress, a static site generator that converts Markdown files into a website.
+- The primary function of `ci/doc.sh` is to generate Markdown files for all documentation.
+
+## Step-by-Step Guide to `ci/doc.sh`
+
+1. **`build_and_convert` Function**:
+   This function generates documentation for shell scripts by invoking `./doc/usage2rst.sh` on all scripts in the specified directory (`$1`). The `usage2rst.sh` script executes each script with the `--help` option and saves the output as an RST file in the `$2/<shell_name>.rst` directory.
+
+2. **Temporary Files Directory**:
+   All temporary files, including RST files, are stored in the `_gen` directory.
+
+3. **`./doc/argparse2rst.py` Script**:
+   This script generates documentation for Python tools located in `espnet/bin`, `espnet2/bin`, and `utils/`. These scripts are executable from the command line, so their documentation is separated from the package information.
+
+4. **`./doc/notebook2rst.sh` Script**:
+   This script generates the demo section by pulling the notebook repository and converting Jupyter Notebook (`.ipynb`) files into Markdown.
+
+5. **`./doc/members2rst.py` Script**:
+   This script generates RST files for all docstrings. It separates out any docstrings for classes or functions that are not class members and excludes private functions (those starting with `_`). The generated RST files are saved in `./_gen/guide`.
+
+6. **Sphinx Build Process**:
+   After copying all necessary files to the `_gen` directory, run `sphinx-build` within `_gen`. Running Sphinx directly in the `doc` directory could cause issues, including potential document corruption. Some files, particularly those ending with `_train` (e.g., `espnet2/bin/asr_train.py`), are excluded from the documentation to avoid errors.
+
+7. **VuePress Directory Setup**:
+   Copy the Markdown files from the `doc` directory, along with files generated in steps 4 and 6, into the `vuepress/src` directory. This is where VuePress recognizes the pages for the site.
+
+8. **Language Support Adjustment**:
+   VuePress doesn’t support some of the programming languages used in code blocks. To address this, we include a command to replace unsupported language codes with equivalent ones.
+
+9. **Generate Navigation Files**:
+   Create the `navbar.yml` and `sidebar.yml` files to define the menus displayed at the top and side of the webpage. For more details, refer to the VuePress-Hope documentation on [navbar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#navbar-config) and [sidebar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#sidebar-config).
+
+10. **Finalize the Build**:
+    Install Node.js and the necessary dependencies, then build the homepage. To preview the page, comment out the `docs:build` line and uncomment the `docs:dev` line in the script.