From 722790dfd3053d5c8d766526014f666df741edc7 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 23 Jul 2024 13:44:07 +0900 Subject: [PATCH 01/57] Added new homepage! - This version uses sphinx and VuePress to automatically generate the modern homepage --- ci/doc.sh | 99 +- doc/argparse2rst.py | 39 +- doc/convert_custom_tags_to_html.py | 230 + doc/convert_md_to_homepage.py | 86 + doc/index.md | 197 + doc/members2rst.py | 101 + doc/notebook2rst.sh | 38 +- doc/vuepress/create_menu.py | 126 + doc/vuepress/package-lock.json | 4775 +++++++++++++++++ doc/vuepress/package.json | 23 + doc/vuepress/src/.vuepress/config.ts | 32 + doc/vuepress/src/.vuepress/head.ts | 10 + doc/vuepress/src/.vuepress/navbar.ts | 7 + ...24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg | 1 + ...24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg | 1 + .../public/assets/image/advanced.svg | 1 + .../.vuepress/public/assets/image/blog.svg | 1 + .../src/.vuepress/public/assets/image/box.svg | 1 + .../assets/image/environment_structure.png | Bin 0 -> 235688 bytes .../.vuepress/public/assets/image/espnet.png | Bin 0 -> 6401 bytes .../public/assets/image/espnet_logo1.png | Bin 0 -> 90599 bytes .../public/assets/image/features.svg | 1 + .../public/assets/image/github-dark.svg | 1 + .../public/assets/image/github-light.svg | 1 + .../.vuepress/public/assets/image/layout.svg | 1 + .../public/assets/image/markdown.svg | 1 + doc/vuepress/src/.vuepress/sidebar.ts | 9 + doc/vuepress/src/.vuepress/styles/config.scss | 3 + doc/vuepress/src/.vuepress/styles/index.scss | 44 + .../src/.vuepress/styles/palette.scss | 3 + doc/vuepress/src/.vuepress/theme.ts | 113 + doc/vuepress/tsconfig.json | 14 + setup.py | 2 + 33 files changed, 5933 insertions(+), 28 deletions(-) create mode 100644 doc/convert_custom_tags_to_html.py create mode 100644 doc/convert_md_to_homepage.py create mode 100644 doc/index.md create mode 100644 doc/members2rst.py create mode 100644 doc/vuepress/create_menu.py create mode 100644 doc/vuepress/package-lock.json create mode 100644 doc/vuepress/package.json create mode 100644 doc/vuepress/src/.vuepress/config.ts create mode 100644 doc/vuepress/src/.vuepress/head.ts create mode 100644 doc/vuepress/src/.vuepress/navbar.ts create mode 100644 doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/advanced.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/blog.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/box.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/environment_structure.png create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/espnet.png create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/espnet_logo1.png create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/features.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/github-light.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/layout.svg create mode 100644 doc/vuepress/src/.vuepress/public/assets/image/markdown.svg create mode 100644 doc/vuepress/src/.vuepress/sidebar.ts create mode 100644 doc/vuepress/src/.vuepress/styles/config.scss create mode 100644 doc/vuepress/src/.vuepress/styles/index.scss create mode 100644 doc/vuepress/src/.vuepress/styles/palette.scss create mode 100644 doc/vuepress/src/.vuepress/theme.ts create mode 100644 doc/vuepress/tsconfig.json diff --git a/ci/doc.sh b/ci/doc.sh index 16653b0a485..b19cf049bc9 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -2,37 +2,110 @@ . tools/activate_python.sh +build_and_convert () { + # $1: path + # $2: output + mkdir -p ./doc/_gen/tools/$2 + for filename in `find $1`; do + bn=`basename ${filename}` + echo "Converting ${filename} to rst..." + ./doc/usage2rst.sh ${filename} > ./doc/_gen/tools/$2/${bn}.rst + done +} + if [ ! -e tools/kaldi ]; then git clone https://github.com/kaldi-asr/kaldi --depth 1 tools/kaldi fi # build sphinx document under doc/ mkdir -p doc/_gen +mkdir -p doc/_gen/tools +mkdir -p doc/_gen/guide # NOTE allow unbound variable (-u) inside kaldi scripts export LD_LIBRARY_PATH=${LD_LIBRARY_PATH-} set -euo pipefail # generate tools doc -( - cd ./utils - ../doc/argparse2rst.py ./*.py > ../doc/_gen/utils_py.rst -) +mkdir utils_py +./doc/argparse2rst.py \ + --title utils_py \ + --output_dir utils_py \ + ./utils/*.py +mv utils_py ./doc/_gen/tools + +mkdir espnet_bin +./doc/argparse2rst.py \ + --title espnet_bin \ + --output_dir espnet_bin \ + ./espnet/bin/*.py +mv espnet_bin ./doc/_gen/tools -./doc/argparse2rst.py ./espnet/bin/*.py > ./doc/_gen/espnet_bin.rst -# FIXME -# ./doc/argparse2rst.py ./espnet2/bin/*.py > ./doc/_gen/espnet2_bin.rst +mkdir espnet2_bin +./doc/argparse2rst.py \ + --title espnet2_bin \ + --output_dir espnet2_bin \ + ./espnet2/bin/*.py +mv espnet2_bin ./doc/_gen/tools +build_and_convert "utils/*.sh" utils +build_and_convert "tools/sentencepiece_commands/spm_*" spm -find ./utils/*.sh tools/sentencepiece_commands/spm_* -exec ./doc/usage2rst.sh {} \; | tee ./doc/_gen/utils_sh.rst -find ./espnet2/bin/*.py -exec ./doc/usage2rst.sh {} \; | tee ./doc/_gen/espnet2_bin.rst +./doc/notebook2rst.sh > ./doc/notebooks.md -./doc/notebook2rst.sh > ./doc/_gen/notebooks.rst +# generate package doc +python ./doc/members2rst.py --root espnet --dst ./doc/_gen/guide --exclude espnet.bin +python ./doc/members2rst.py --root espnet2 --dst ./doc/_gen/guide --exclude espnet2.bin +python ./doc/members2rst.py --root espnetez --dst ./doc/_gen/guide # generate package doc ./doc/module2rst.py --root espnet espnet2 --dst ./doc --exclude espnet.bin -# build html -# TODO(karita): add -W to turn warnings into errors -sphinx-build -b html doc doc/build +# build markdown +cp ./doc/index.rst ./doc/_gen/index.rst +cp ./doc/conf.py ./doc/_gen/ +rm -f ./doc/_gen/tools/espnet2_bin/*_train.rst +sphinx-build -M markdown ./doc/_gen ./doc/build + +# copy markdown files to specific directory. +cp -r ./doc/build/markdown/* ./doc/vuepress/src/ +cp -r ./doc/notebook ./doc/vuepress/src/ +cp ./doc/*.md ./doc/vuepress/src/ +mv ./doc/vuepress/src/README.md ./doc/vuepress/src/document.md +cp -r ./doc/image ./doc/vuepress/src/ + +# Document generation has finished. +# From the following point we modify files for VuePress. +# Replace language tags to supported language tags +find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```default/```text/g' {} \; +find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```pycon/```python/g' {} \; +find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```cd/```text/g' {} \; + +# And convert custom tags to < and >, as can be recognized a html tag. +python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/ + +# Convert API document to specific html tags to display sphinx style +python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/guide/ +python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/tools/ + +# Create navbar and sidebar. +cd ./doc/vuepress +python create_menu.py --root ./src + +# check if node is installed +if which node > /dev/null +then + echo "node is installed, skipping..." +else + apt install -y nodejs npm + npm install n -g + n stable + apt purge -y nodejs npm + apt autoremove -y +fi + +npm i +# npm run docs:dev +npm run docs:build +mv src/.vuepress/dist ../../ touch doc/build/.nojekyll diff --git a/doc/argparse2rst.py b/doc/argparse2rst.py index 684673d90a3..d10a00acbd0 100755 --- a/doc/argparse2rst.py +++ b/doc/argparse2rst.py @@ -3,6 +3,7 @@ import logging import pathlib import re +import os import configargparse @@ -24,6 +25,18 @@ def get_parser(): config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter, ) + parser.add_argument( + "--title", + type=str, + default="Module Reference", + help="title for the generated RST", + ) + parser.add_argument( + "--output_dir", + type=str, + default=".", + help="output directory to save generated RSTs", + ) parser.add_argument( "src", type=str, @@ -42,10 +55,17 @@ def get_parser(): for p in args.src: if "__init__.py" in p: continue - modinfo.append(ModuleInfo(p)) + try: + modinfo.append(ModuleInfo(p)) + except Exception as e: + logging.error(f"Error processing {p}: {str(e)}") +print(f""" +{args.title} +{"=" * len(args.title)} + +""") -# print refs for m in modinfo: logging.info(f"processing: {m.path.name}") d = m.module.get_parser().description @@ -54,15 +74,15 @@ def get_parser(): print() -# print argparse +os.makedirs(args.output_dir, exist_ok=True) + +# print argparse to each files for m in modinfo: cmd = m.path.name sep = "~" * len(cmd) - print( - f""" - -.. _{cmd}: - + with open(f"{args.output_dir}/{cmd[:-3]}.rst", "w") as writer: # remove .py + writer.write( + f""".. _{cmd} {cmd} {sep} @@ -71,5 +91,4 @@ def get_parser(): :func: get_parser :prog: {cmd} -""" - ) +""") diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py new file mode 100644 index 00000000000..175f6db0470 --- /dev/null +++ b/doc/convert_custom_tags_to_html.py @@ -0,0 +1,230 @@ +import os +import glob +import re +import configargparse + +ALL_HTML_TAGS = [ + "a", + "abbr", + "acronym", + "address", + "applet", + "area", + "article", + "aside", + "audio", + "b", + "base", + "basefont", + "bdi", + "bdo", + "big", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "center", + "cite", + "code", + "col", + "colgroup", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "dir", + "div", + "dl", + "dt", + "em", + "embed", + "fieldset", + "figcaption", + "figure", + "font", + "footer", + "form", + "frame", + "frameset", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "hr", + "html", + "i", + "iframe", + "img", + "input", + "ins", + "kbd", + "label", + "legend", + "li", + "link", + "main", + "map", + "mark", + "menu", + "meta", + "meter", + "nav", + "noframes", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "param", + "picture", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "search", + "section", + "select", + "small", + "source", + "span", + "strike", + "strong", + "style", + "sub", + "summary", + "sup", + "svg", + "table", + "tbody", + "td", + "template", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "tt", + "u", + "ul", + "var", + "video", + "wbr", +] + +def get_parser(): + parser = configargparse.ArgumentParser( + description="Convert custom tags to markdown", + config_file_parser_class=configargparse.YAMLConfigFileParser, + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "root", + type=str, + help="source python files that contain get_parser() func", + ) + return parser + + +def replace_custom_tags(content): + # Regex to find tags and their content + tag_pattern = re.compile(r'<(?!\/)(?!\!!-!-)(.+?)(?!\/)>') + def replace_tag(match): + tag_name = match.group(1) + if len(tag_name) > 50: + # heuristics to ignore tags with too long names + # This might occur with image tags, since they have image data + # in base64 format. + return match.group(0) + + if ( + tag_name.split()[0] not in ALL_HTML_TAGS + or ( + len(tag_name.split()) > 1 and "=" not in tag_name + ) + ): + return f"<{tag_name}>" + + end_tag_pattern = re.compile(f'') + end_tag_match = end_tag_pattern.search(content, match.end()) + if not end_tag_match: + return f"<{tag_name}>" + return match.group(0) + return tag_pattern.sub(replace_tag, content) + + +def replace_string_tags(content): + # Regex to find tags and their content + tag_pattern = re.compile(r"['|\"]<(?!\/)(.+?)(?!\/)>['|\"]") + def replace_tag(match): + tag_name = match.group(1) + if len(tag_name) > 50: + # heuristics to ignore tags with too long names + # This might occur with image tags, since they have image data + # in base64 format. + return match.group(0) + if ( + tag_name.split()[0] not in ALL_HTML_TAGS + or ( + len(tag_name.split()) > 1 and "=" not in tag_name + ) + ): + return f"'<{tag_name}>'" + + end_tag_pattern = re.compile(f'') + end_tag_match = end_tag_pattern.search(content, match.end()) + if not end_tag_match: + return f"'<{tag_name}>'" + return match.group(0) + return tag_pattern.sub(replace_tag, content) + + +# parser +args = get_parser().parse_args() + +for md in glob.glob(f"{args.root}/*.md", recursive=True): + with open(md, "r") as f: + content = f.read() + + # Replace the "" and "" with "<" and ">", respectively + # if the tag is not in ALL_HTML_TAGS and does not have its end tag + # we need to apply this two functions because + # there are custom tags like: "" + content = replace_string_tags(content) + content = replace_custom_tags(content) + + with open(md, "w") as f: + f.write(content) + + +for md in glob.glob(f"{args.root}/**/*.md", recursive=True): + with open(md, "r") as f: + content = f.read() + + # Replace the "" and "" with "<" and ">", respectively + # if the tag is not in ALL_HTML_TAGS + content = replace_string_tags(content) + content = replace_custom_tags(content) + + with open(md, "w") as f: + f.write(content) + diff --git a/doc/convert_md_to_homepage.py b/doc/convert_md_to_homepage.py new file mode 100644 index 00000000000..c9372877499 --- /dev/null +++ b/doc/convert_md_to_homepage.py @@ -0,0 +1,86 @@ +import markdown +import re +import copy +import configargparse +from glob import glob + + +def get_parser(): + parser = configargparse.ArgumentParser( + description="Convert custom tags to markdown", + config_file_parser_class=configargparse.YAMLConfigFileParser, + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "root", + type=str, + help="source markdown file", + ) + return parser + + +def small_bracket(text): + # forward_core(nnet_output, ys, hlens, ylens) + # -> forward_core(nnet_output, ys, hlens, ylens)" + text = text.replace("<", "<").replace(">", ">") + brackets = re.findall(r'\(([^)]+)\)', text) + if len(brackets) > 0: + text = text.replace( + f"({brackets[0]})", + f"({brackets[0]})" + ) + return text + + +def convert(markdown_text): + # convert "#### Examples" to :::note ::: block + # We assume that this example block will continue to the next header. + example_pattern = r'###\sExamples' + _result = copy.copy(markdown_text) + for match in re.finditer(example_pattern, _result): + _result = _result.replace( + match.group(0), + f"##### Examples" + ) + + # convert ### to div with specific class + h3_pattern = re.compile(r'^###\s+(.+)$', re.MULTILINE) + for match in re.finditer(h3_pattern, _result): + tag_removed = match.group(0).replace("### ", "") + tag_removed = small_bracket(tag_removed) + tag_removed = markdown.markdown(tag_removed) + _result = _result.replace( + match.group(0), + f"
{tag_removed}
\n" + ) + + # convert "#### Note" to :::note ::: block + # We assume that this note block will continue to the next header. + note_pattern = r'####\sNOTE' + for match in re.finditer(note_pattern, _result): + _result = _result.replace( + match.group(0), + f"##### NOTE" + ) + + # Convert "####" to custom-h4 tag. + h4_pattern = re.compile(r'^####\s+(.+)$', re.MULTILINE) + for match in re.finditer(h4_pattern, _result): + tag_removed = match.group(0).replace("#### ", "") + tag_removed = small_bracket(tag_removed) + tag_removed = markdown.markdown(tag_removed) + _result = _result.replace( + match.group(0), + f"
{tag_removed}
\n" + ) + return _result + + +# parser +args = get_parser().parse_args() + +for md in glob(f"{args.root}/**/*.md", recursive=True): + markdown_text = open(md, "r").read() + _result = convert(markdown_text) + with open(md, "w") as f: + f.write(_result) diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 00000000000..ac7416fb4e6 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,197 @@ +git s--- +home: true +icon: /assets/image/espnet.png +title: ESPnet +heroImage: /assets/image/espnet_logo1.png +heroImageStyle: + - width: 80% +heroText: "ESPnet: end-to-end speech processing toolkit" +tagline: "ESPnet is an end-to-end speech processing toolkit covering many speech-related tasks." +actions: + - text: Get Started + icon: book + link: ./espnet2_tutorial.md + type: primary + + - text: Demos + icon: lightbulb + link: ./notebook/ + + +highlights: + - header: Easy to install + image: /assets/image/box.svg + bgImage: https://theme-hope-assets.vuejs.press/bg/3-light.svg + bgImageDark: https://theme-hope-assets.vuejs.press/bg/3-dark.svg + highlights: + - title: pip install espnet for easy install. + - title: See this instruction for more details. + + - header: Supports many tasks + description: We provide a complete setup for various speech processing tasks. + image: https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEj3mOiQTPh_S9XW6m94OQYjucUzUu7L9uEcHP9YsADUGWTcmscynkrLc1Zs8o5rA3G9lSNnEpyHBMCnZzBepYdW8jVofKnLflvOsu-ywIZpQf1Kw5l6tzvhEA1q2cbnFDIzIDlOUOKPOarf/s800/cooking_recipe.png + bgImage: https://theme-hope-assets.vuejs.press/bg/2-light.svg + bgImageDark: https://theme-hope-assets.vuejs.press/bg/2-dark.svg + bgImageStyle: + background-repeat: repeat + background-size: initial + features: + - title: "ASR: Automatic Speech Recognition" + link: https://github.com/espnet/espnet?tab=readme-ov-file#asr-automatic-speech-recognition + + - title: "TTS: Text-to-speech" + link: https://github.com/espnet/espnet?tab=readme-ov-file#tts-text-to-speech + + - title: "SE: Speech enhancement (and separation)" + link: https://github.com/espnet/espnet?tab=readme-ov-file#se-speech-enhancement-and-separation + + - title: "SUM: Speech Summarization" + link: https://github.com/espnet/espnet?tab=readme-ov-file#sum-speech-summarization + + - title: "SVS: Singing Voice Synthesis" + link: https://github.com/espnet/espnet?tab=readme-ov-file#svs-singing-voice-synthesis + + - title: "SSL: Self-supervised Learning" + link: https://github.com/espnet/espnet?tab=readme-ov-file#ssl-self-supervised-learning + + - title: "UASR: Unsupervised ASR " + details: "EURO: ESPnet Unsupervised Recognition - Open-source" + link: https://github.com/espnet/espnet?tab=readme-ov-file#uasr-unsupervised-asr-euro-espnet-unsupervised-recognition---open-source + + - title: "S2T: Speech-to-text with Whisper-style multilingual multitask models" + link: https://github.com/espnet/espnet?tab=readme-ov-file#s2t-speech-to-text-with-whisper-style-multilingual-multitask-models + + - header: More Documents + description: You can find tutorials on ESPnet packages. + bgImage: https://theme-hope-assets.vuejs.press/bg/10-light.svg + bgImageDark: https://theme-hope-assets.vuejs.press/bg/10-dark.svg + bgImageStyle: + background-repeat: repeat + background-size: initial + highlights: + - title: Tutorial + icon: /assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg + link: ./tutorial.md + + - title: Tutorial (ESPnet1) + icon: /assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg + link: ./espnet1_tutorial.md + + - title: Training Config + icon: sliders + link: ./espnet2_training_option.md + + - title: Format audio to wav.scp + icon: arrow-rotate-right + link: ./espnet2_format_wav_scp.md + + - title: Task class and data + icon: database + link: ./espnet2_task.md + + - title: Docker + icon: /assets/icon/docker-mark-blue.svg + link: ./docker.md + + - title: Job scheduling system + icon: server + link: ./parallelization.md + + - title: Distributed training + icon: server + link: ./espnet2_distributed.md + + - title: Document Generation + icon: book + link: ./document.md + +footer: Apache License 2.0, Copyright © 2024-present ESPnet community +--- + +## Citations + +``` +@inproceedings{watanabe2018espnet, + author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson {Enrique Yalta Soplin} and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai}, + title={{ESPnet}: End-to-End Speech Processing Toolkit}, + year={2018}, + booktitle={Proceedings of Interspeech}, + pages={2207--2211}, + doi={10.21437/Interspeech.2018-1456}, + url={http://dx.doi.org/10.21437/Interspeech.2018-1456} +} +@inproceedings{hayashi2020espnet, + title={{Espnet-TTS}: Unified, reproducible, and integratable open source end-to-end text-to-speech toolkit}, + author={Hayashi, Tomoki and Yamamoto, Ryuichi and Inoue, Katsuki and Yoshimura, Takenori and Watanabe, Shinji and Toda, Tomoki and Takeda, Kazuya and Zhang, Yu and Tan, Xu}, + booktitle={Proceedings of IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, + pages={7654--7658}, + year={2020}, + organization={IEEE} +} +@inproceedings{inaguma-etal-2020-espnet, + title = "{ESP}net-{ST}: All-in-One Speech Translation Toolkit", + author = "Inaguma, Hirofumi and + Kiyono, Shun and + Duh, Kevin and + Karita, Shigeki and + Yalta, Nelson and + Hayashi, Tomoki and + Watanabe, Shinji", + booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations", + month = jul, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.acl-demos.34", + pages = "302--311", +} +@article{hayashi2021espnet2, + title={Espnet2-tts: Extending the edge of tts research}, + author={Hayashi, Tomoki and Yamamoto, Ryuichi and Yoshimura, Takenori and Wu, Peter and Shi, Jiatong and Saeki, Takaaki and Ju, Yooncheol and Yasuda, Yusuke and Takamichi, Shinnosuke and Watanabe, Shinji}, + journal={arXiv preprint arXiv:2110.07840}, + year={2021} +} +@inproceedings{li2020espnet, + title={{ESPnet-SE}: End-to-End Speech Enhancement and Separation Toolkit Designed for {ASR} Integration}, + author={Chenda Li and Jing Shi and Wangyou Zhang and Aswin Shanmugam Subramanian and Xuankai Chang and Naoyuki Kamo and Moto Hira and Tomoki Hayashi and Christoph Boeddeker and Zhuo Chen and Shinji Watanabe}, + booktitle={Proceedings of IEEE Spoken Language Technology Workshop (SLT)}, + pages={785--792}, + year={2021}, + organization={IEEE}, +} +@inproceedings{arora2021espnet, + title={{ESPnet-SLU}: Advancing Spoken Language Understanding through ESPnet}, + author={Arora, Siddhant and Dalmia, Siddharth and Denisov, Pavel and Chang, Xuankai and Ueda, Yushi and Peng, Yifan and Zhang, Yuekai and Kumar, Sujay and Ganesan, Karthik and Yan, Brian and others}, + booktitle={ICASSP 2022-2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, + pages={7167--7171}, + year={2022}, + organization={IEEE} +} +@inproceedings{shi2022muskits, + author={Shi, Jiatong and Guo, Shuai and Qian, Tao and Huo, Nan and Hayashi, Tomoki and Wu, Yuning and Xu, Frank and Chang, Xuankai and Li, Huazhe and Wu, Peter and Watanabe, Shinji and Jin, Qin}, + title={{Muskits}: an End-to-End Music Processing Toolkit for Singing Voice Synthesis}, + year={2022}, + booktitle={Proceedings of Interspeech}, + pages={4277-4281}, + url={https://www.isca-speech.org/archive/pdfs/interspeech_2022/shi22d_interspeech.pdf} +} +@inproceedings{lu22c_interspeech, + author={Yen-Ju Lu and Xuankai Chang and Chenda Li and Wangyou Zhang and Samuele Cornell and Zhaoheng Ni and Yoshiki Masuyama and Brian Yan and Robin Scheibler and Zhong-Qiu Wang and Yu Tsao and Yanmin Qian and Shinji Watanabe}, + title={{ESPnet-SE++: Speech Enhancement for Robust Speech Recognition, Translation, and Understanding}}, + year=2022, + booktitle={Proc. Interspeech 2022}, + pages={5458--5462}, +} +@article{gao2022euro, + title={{EURO}: {ESPnet} Unsupervised ASR Open-source Toolkit}, + author={Gao, Dongji and Shi, Jiatong and Chuang, Shun-Po and Garcia, Leibny Paola and Lee, Hung-yi and Watanabe, Shinji and Khudanpur, Sanjeev}, + journal={arXiv preprint arXiv:2211.17196}, + year={2022} +} +@article{peng2023reproducing, + title={Reproducing Whisper-Style Training Using an Open-Source Toolkit and Publicly Available Data}, + author={Peng, Yifan and Tian, Jinchuan and Yan, Brian and Berrebbi, Dan and Chang, Xuankai and Li, Xinjian and Shi, Jiatong and Arora, Siddhant and Chen, William and Sharma, Roshan and others}, + journal={arXiv preprint arXiv:2309.13876}, + year={2023} +} +``` \ No newline at end of file diff --git a/doc/members2rst.py b/doc/members2rst.py new file mode 100644 index 00000000000..579f7b6a8b9 --- /dev/null +++ b/doc/members2rst.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +from glob import glob +import importlib +import os +import ast +import sys + +import configargparse + + +def to_module(path_name): + ret = path_name.replace(".py", "").replace("/", ".") + if ret.endswith("."): + return ret[:-1] + return ret + + +def top_level_functions(body): + return (f for f in body + if isinstance(f, ast.FunctionDef) + and not f.name.startswith("_") + ) + + +def top_level_classes(body): + return (f for f in body if isinstance(f, ast.ClassDef)) + + +def parse_ast(filename): + with open(filename, "rt") as file: + return ast.parse(file.read(), filename=filename) + + +def gen_func_rst(func_name, writer): + writer.write(f""".. _{func_name} +{func_name} +{"~" * len(func_name)} + +.. autofunction:: {func_name} +""") + + +def gen_class_rst(class_name, writer): + writer.write(f""".. _{class_name} +{class_name} +{"~" * len(class_name)} + +.. autoclass:: {class_name} + :members: + :undoc-members: + :show-inheritance: +""") + +# parser +parser = configargparse.ArgumentParser( + description="generate RST files from module recursively into /_gen", + config_file_parser_class=configargparse.YAMLConfigFileParser, + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, +) +parser.add_argument( + "--root", type=str, help="root module to generate docs" +) +parser.add_argument("--dst", type=str, help="destination path to generate RSTs") +parser.add_argument("--exclude", nargs="*", default=[], help="exclude module name") +args = parser.parse_args() +print(args) + + +gendir = args.dst +os.makedirs(gendir, exist_ok=True) +os.makedirs(f"{gendir}/{args.root}", exist_ok=True) + +for p in glob(args.root + "/**", recursive=True): + module_name = to_module(p) + if any([ex in module_name for ex in args.exclude]): + continue + if "__init__" in p: + continue + if not p.endswith(".py"): + continue + + submodule_name = module_name.split(".")[1] + os.makedirs(f"{gendir}/{args.root}/{submodule_name}", exist_ok=True) + + if not os.path.exists(f"{gendir}/{args.root}/{submodule_name}/README.rst"): + # 1 get functions + for func in top_level_functions(parse_ast(p).body): + function_name = func.name + print(f"[INFO] generating {func.name} in {module_name}") + # 1.2 generate RST + with open(f"{gendir}/{args.root}/{submodule_name}/{function_name}.rst", "w") as f_rst: + gen_func_rst(f"{module_name}.{function_name}", f_rst) + + # 2 get classes + for clz in top_level_classes(parse_ast(p).body): + class_name = clz.name + print(f"[INFO] generating {clz.name} in {module_name}") + # 1.2 generate RST + with open(f"{gendir}/{args.root}/{submodule_name}/{class_name}.rst", "w") as f_rst: + gen_class_rst(f"{module_name}.{class_name}", f_rst) + diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index 83bf7d57794..c9d3f5c6762 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -8,10 +8,38 @@ if [ ! -d notebook ]; then git clone https://github.com/espnet/notebook --depth 1 fi -echo "\ -.. toctree:: - :maxdepth: 1 - :caption: Notebook: +echo "# Notebook + +Jupyter notebooks for course demos and tutorials. " -find ./notebook/*.ipynb -exec echo " {}" \; +cd notebook +documents=`for i in $(ls -d */); do echo ${i%%/}; done` +for document in "${documents[@]}"; do + echo "## ${document}\n" + find ./${document} \ + -type f \ + -name '*.ipynb' \ + -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --clear-output \"{}\"" \; + find ./${document} \ + -type f \ + -name '*.ipynb' \ + -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --to markdown \"{}\"" \; + + basedir=./${document} + for md_file in `find "./${document}" -name "*.md"`; do + filename=`basename ${md_file}` + echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" + done + for ipynb_file in `find "./${document}" -name "*.ipynb"`; do + rm ${ipynb_file} + done + + # generate README.md + echo "# ${document} Demo" > ./${document}/README.md + for md_file in `find "./${document}" -name "*.md"`; do + filename=`basename ${md_file}` + echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" >> ./${document}/README.md + done + echo "" +done diff --git a/doc/vuepress/create_menu.py b/doc/vuepress/create_menu.py new file mode 100644 index 00000000000..8ddd1c357e5 --- /dev/null +++ b/doc/vuepress/create_menu.py @@ -0,0 +1,126 @@ +import os +import glob +import argparse +import yaml + + +def get_menubar_recursively(directory): + menubars = [] + print(f'Scanning directory: {directory}') + for child in glob.glob(os.path.join(directory, '*')): + if os.path.isdir(child): + children = get_menubar_recursively(child) + if len(children) > 0: + # menubars[0]['children'].append(children[0]['children']) + menubars.append({ + 'text': child.lower().split('/')[-1].upper(), + 'children': get_menubar_recursively(child) + }) + else: + if os.path.splitext(child)[1].lower() == '.ipynb': + menubars.append(f'/{child[len(DOCS):-6]}') # remoce '.ipynb' + elif os.path.splitext(child)[1].lower() == '.md': + menubars.append(f'/{child[len(DOCS):-3]}') # remoce '.ipynb' + return menubars + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--root', required=True, + type=str, help='List of directory') + args = parser.parse_args() + + navbars = [] + sidebars = [] + + DOCS = args.root + '/' + + # 1. Create navBar + # 1.1. Create guide + navbars.append({ + 'text': "Guide", + 'icon': "book", + 'prefix': f"guide/", + 'children': [] + }) + + for doc in glob.glob(f"{DOCS}/guide/**/"): + doc_name = doc.split("/")[-2] + navbars[0]['children'].append({ + 'text': doc_name, + 'prefix': f"{doc_name}/", + 'children': [f"README.md"] + sorted([ + f"{submodule.split('/')[-2]}/README.md" + for submodule in glob.glob(f"{doc}/**/") + ]) + }) + + # 1.2. Create Tools + navbars.append({ + 'text': "Tools", + 'icon': "wrench", + 'prefix': f"tools/", + 'children': [] + }) + + for doc in glob.glob(f"{DOCS}/tools/**/"): + doc = doc.split("/")[-2] + navbars[1]['children'].append({ + 'text': doc, + 'prefix': f"{doc}/", + 'children': [f"README.md"] + }) + + # 1.2. Create Notebooks + navbars.append({ + 'text': "Demo", + 'icon': "laptop-code", + 'prefix': f"notebook/", + 'children': [] + }) + + for doc in glob.glob(f"{DOCS}/notebook/**/"): + doc = doc.split("/")[-2] + navbars[2]['children'].append({ + 'text': doc, + 'prefix': f"{doc}/", + 'children': [f"README.md"] + }) + + # 1.2.1. sort + navbars[1]['children'].sort(key=lambda x: x['text'].lower()) + navbars[2]['children'].sort(key=lambda x: x['text'].lower()) + + # 1.3 write navBars.yml + with open('navbars.yml', 'w', encoding='utf-8') as f: + yaml.dump(navbars, f, default_flow_style=False) + + # 2. Create sidebars + # 2.1. Create guide + sidebars.append({ + "text": "Guide", + "icon": "book", + "prefix": "guide/", + "children": "structure", + }) + + # 2.2. Create Tools + sidebars.append({ + "text": "Tools", + "icon": "wrench", + "prefix": "tools/", + "children": "structure", + }) + + # 2.3. Create Notebooks + sidebars.append({ + "text": "Demo", + "icon": "laptop-code", + "prefix": "notebook/", + "children": "structure", + }) + + # 2.3. Write sidebars.yml + with open('sidebars.yml', 'w', encoding='utf-8') as f: + yaml.dump(sidebars, f, default_flow_style=False) + diff --git a/doc/vuepress/package-lock.json b/doc/vuepress/package-lock.json new file mode 100644 index 00000000000..8847e70bcee --- /dev/null +++ b/doc/vuepress/package-lock.json @@ -0,0 +1,4775 @@ +{ + "name": "espnet_page", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "espnet_page", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "js-yaml": "^4.1.0" + }, + "devDependencies": { + "@vuepress/bundler-vite": "2.0.0-rc.14", + "vue": "^3.4.31", + "vuepress": "2.0.0-rc.14", + "vuepress-plugin-search-pro": "^2.0.0-rc.51", + "vuepress-theme-hope": "2.0.0-rc.51" + } + }, + "node_modules/@babel/parser": { + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.8.tgz", + "integrity": "sha512-WzfbgXOkGzZiXXCqk43kKwZjzwx4oulxZi3nq2TYL9mOjQv6kYwul9mz6ID36njuL7Xkp6nJEfok848Zj10j/w==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true + }, + "node_modules/@lit-labs/ssr-dom-shim": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.2.0.tgz", + "integrity": "sha512-yWJKmpGE6lUURKAaIltoPIE/wrbY3TEkqQt+X0m+7fQNnAv0keydnYvbiJFP1PnMhizmIWRWOG5KLhYyc/xl+g==", + "dev": true + }, + "node_modules/@lit/reactive-element": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-2.0.4.tgz", + "integrity": "sha512-GFn91inaUa2oHLak8awSIigYz0cU0Payr1rcFsrkf5OJ5eSPxElyZfKh0f2p9FsTiZWXQdWGJeXZICEfXXYSXQ==", + "dev": true, + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.2.0" + } + }, + "node_modules/@mdit-vue/plugin-component": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-component/-/plugin-component-2.1.3.tgz", + "integrity": "sha512-9AG17beCgpEw/4ldo/M6Y/1Rh4E1bqMmr/rCkWKmCAxy9tJz3lzY7HQJanyHMJufwsb3WL5Lp7Om/aPcQTZ9SA==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/plugin-frontmatter": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-frontmatter/-/plugin-frontmatter-2.1.3.tgz", + "integrity": "sha512-KxsSCUVBEmn6sJcchSTiI5v9bWaoRxe68RBYRDGcSEY1GTnfQ5gQPMIsM48P4q1luLEIWurVGGrRu7u93//LDQ==", + "dev": true, + "dependencies": { + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "gray-matter": "^4.0.3", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/plugin-headers": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-headers/-/plugin-headers-2.1.3.tgz", + "integrity": "sha512-AcL7a7LHQR3ISINhfjGJNE/bHyM0dcl6MYm1Sr//zF7ZgokPGwD/HhD7TzwmrKA9YNYCcO9P3QmF/RN9XyA6CA==", + "dev": true, + "dependencies": { + "@mdit-vue/shared": "2.1.3", + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/plugin-sfc": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-sfc/-/plugin-sfc-2.1.3.tgz", + "integrity": "sha512-Ezl0dNvQNS639Yl4siXm+cnWtQvlqHrg+u+lnau/OHpj9Xh3LVap/BSQVugKIV37eR13jXXYf3VaAOP1fXPN+w==", + "dev": true, + "dependencies": { + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/plugin-title": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-title/-/plugin-title-2.1.3.tgz", + "integrity": "sha512-XWVOQoZqczoN97xCDrnQicmXKoqwOjIymIm9HQnRXhHnYKOgJPW1CxSGhkcOGzvDU1v0mD/adojVyyj/s6ggWw==", + "dev": true, + "dependencies": { + "@mdit-vue/shared": "2.1.3", + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/plugin-toc": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/plugin-toc/-/plugin-toc-2.1.3.tgz", + "integrity": "sha512-41Q+iXpLHZt0zJdApVwoVt7WF6za/xUjtjEPf90Z3KLzQO01TXsv48Xp9BsrFHPcPcm8tiZ0+O1/ICJO80V/MQ==", + "dev": true, + "dependencies": { + "@mdit-vue/shared": "2.1.3", + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/shared": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@mdit-vue/shared/-/shared-2.1.3.tgz", + "integrity": "sha512-27YI8b0VVZsAlNwaWoaOCWbr4eL8B04HxiYk/y2ktblO/nMcOEOLt4p0RjuobvdyUyjHvGOS09RKhq7qHm1CHQ==", + "dev": true, + "dependencies": { + "@mdit-vue/types": "2.1.0", + "@types/markdown-it": "^14.1.1", + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit-vue/types": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@mdit-vue/types/-/types-2.1.0.tgz", + "integrity": "sha512-TMBB/BQWVvwtpBdWD75rkZx4ZphQ6MN0O4QB2Bc0oI5PC2uE57QerhNxdRZ7cvBHE2iY2C+BUNUziCfJbjIRRA==", + "dev": true + }, + "node_modules/@mdit/plugin-alert": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-alert/-/plugin-alert-0.12.0.tgz", + "integrity": "sha512-4OyGK1PZrJbmEF/kS6GKmmG1nlN5h/CyIPZV8lRgnlWLFB37JiEz3EHusPAXAoMtw7VGNFaIcl7OT/I5yyz1JQ==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-align": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-align/-/plugin-align-0.12.0.tgz", + "integrity": "sha512-rvA+xzaVrlsr44s7XD/xadO3lF0QYWCbeSrOS2dhOroNCIOy4RotVP/1tQPr84eqm4oXcxXF0cbjFuwUgE1jYw==", + "dev": true, + "dependencies": { + "@mdit/plugin-container": "0.12.0", + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-attrs": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-attrs/-/plugin-attrs-0.12.0.tgz", + "integrity": "sha512-J0MBwBq958lBtdIcEo02mUIO4ubl2YK+bY799T2SusrLTf3FZsq8+d/OiLTUtovfxaphD7F6yqo8M61AiOpq+w==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-container": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-container/-/plugin-container-0.12.0.tgz", + "integrity": "sha512-61bWK1ek6Rn4o12/BIKTWgGU0miB9ENcXE19H5D4DRhwG5+4+0zp2U6hRLf/mE73+mRYin7iKVzcwwEsqs+u8w==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-demo": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-demo/-/plugin-demo-0.12.0.tgz", + "integrity": "sha512-+KDUOgcvnMtBN/uYWlhIFuWkTJexuxstq8ERy9q7vOiu8Go85qCb27h0RSToKBTmmGy+XqfU2EdJclYPWBupJQ==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-figure": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-figure/-/plugin-figure-0.12.0.tgz", + "integrity": "sha512-3nfcGI+uM0f6AqHZrEr8kSMBI6T2+fKKQXtCbvWQqQ+P3iGgf34Ay2eAtuMDcDGqyfNuR6e8aLoOeY2QWuEynA==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-footnote": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-footnote/-/plugin-footnote-0.12.0.tgz", + "integrity": "sha512-9B+bJdMndCPoA9De9bxRm4/fyz02PHRcttOyuyPJ3G+wCAgIN1c/7CB8ViT1YJuECUjLogJQ/rrgqh7f0LTqLQ==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + } + }, + "node_modules/@mdit/plugin-img-lazyload": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-img-lazyload/-/plugin-img-lazyload-0.12.0.tgz", + "integrity": "sha512-6R42ieXzwkB5BKKZi+ZefqeP/fBG5qo7Sqtl72ewSVqEQ30bgxpk6nkrPI2orRob4tb6z0F/c+R8h6PW5MkTOw==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-img-mark": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-img-mark/-/plugin-img-mark-0.12.0.tgz", + "integrity": "sha512-HkIUwlTg/xPsBi4PG+5dsMnsb7wdiJzELSCEUfdAJTg55nksonHfyV2pFpr87MML4nuZlZK9JHt+Bm2BBDSVSw==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-img-size": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-img-size/-/plugin-img-size-0.12.0.tgz", + "integrity": "sha512-fCcF5gc+ba6gQ5ebrKuI8bK/gFbj8mbeN45FHmBsFDFsfTHa0Xij2v8iok0nP8YEIVj71y8XYojsqCWs6avong==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-include": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-include/-/plugin-include-0.12.0.tgz", + "integrity": "sha512-8pnmp7s1TjbtoBIa/YhYpEivOpeVSyhkQoQrGq1UoaEcTbXqmFwShGkAW3zUYZVFYTl74PgL/UqJnrUojegJQg==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1", + "upath": "^2.0.1" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-katex-slim": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-katex-slim/-/plugin-katex-slim-0.12.0.tgz", + "integrity": "sha512-s2MJGXFZT7u8IUTmy6K1rxxAdYRmGggu0m860siyUrThL112xLN9r3jmXZ83epgi4UA/gLkRDAU5vF6R2JtyjQ==", + "dev": true, + "dependencies": { + "@mdit/plugin-tex": "0.12.0", + "@types/katex": "^0.16.7", + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "katex": "^0.16.9", + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "katex": { + "optional": true + }, + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-mark": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-mark/-/plugin-mark-0.12.0.tgz", + "integrity": "sha512-BDFwbV/tbgUGL8KF2ymYNLEXT2KNBLe8D0rshDrbB4Iko1U2DywACQkmaUbYBJ1VCn7/dff35at9fWrm3QjrwQ==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-mathjax-slim": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-mathjax-slim/-/plugin-mathjax-slim-0.12.0.tgz", + "integrity": "sha512-bLM+JnCTN/3XiyKb64Yhpx014VYLfHBexua4n92cUyoKR9g3waB0loF1WMlg6GdyCTc7OvrUSceNjwWj3YRogg==", + "dev": true, + "dependencies": { + "@mdit/plugin-tex": "0.12.0", + "@types/markdown-it": "^14.1.1", + "upath": "^2.0.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0", + "mathjax-full": "^3.2.2" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + }, + "mathjax-full": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-plantuml": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-plantuml/-/plugin-plantuml-0.12.0.tgz", + "integrity": "sha512-m1pk6PA9+kWUs8kylLqjnQ7Lex68x3c4Ato8zAh+omkhugfWzuQXfFiXRiJ9C7wkdqHoJx/E5XobP3HJnhCpoA==", + "dev": true, + "dependencies": { + "@mdit/plugin-uml": "0.12.0", + "@types/markdown-it": "^14.1.1" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-spoiler": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-spoiler/-/plugin-spoiler-0.12.0.tgz", + "integrity": "sha512-7yu+Gz000O0OxGnGYOoj77Am3WgH4GwzOvwCp7tPLexkJwTve8MyT9In/NEPFaRw8fmgXwthC0gKq4Ubh1+8DA==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-stylize": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-stylize/-/plugin-stylize-0.12.0.tgz", + "integrity": "sha512-5bzZvmjEpGTdwBax9jaDbCBhD1snEx6uTHVUG9HD/L5koKrL86+ox9E5FGeiMiD1dtxeMgL+WqBzV44nRE9ZPg==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-sub": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-sub/-/plugin-sub-0.12.0.tgz", + "integrity": "sha512-27kKkSVkymc+2RNc5XOYkeXip5PgHZPUnHpxUvkpnairLwyHsXb8/gzr9zd5arVkip86rcdy9LIvnF7zO0dNVQ==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-sup": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-sup/-/plugin-sup-0.12.0.tgz", + "integrity": "sha512-3bEDW5/y1UDVU8LVbFsqUvNcMW6orp16uCdRGYCNZ3/IeK7Qj1/9a3wfhScIoI8xRUE6M3JLv41sGBFXLHwi1w==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-tab": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-tab/-/plugin-tab-0.12.0.tgz", + "integrity": "sha512-ZDTEDxHoekcFA5Al+NLizn8Nf0kj6ABkNBAc/VxbQoVQdjZNQtGY2dOPeWW0I96Rao+Aw+IpYRCLFIfb/KtExw==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-tasklist": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-tasklist/-/plugin-tasklist-0.12.0.tgz", + "integrity": "sha512-MPmuLJrqHYR2xI7ST9Xtw/xj+6Xoq7kUvcGuXWdMMNT11DcU1KppkR8QBHov437NFYh6aGyjrHUVeM4T5Ls8yg==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-tex": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-tex/-/plugin-tex-0.12.0.tgz", + "integrity": "sha512-ejeSgSeZvcI5P4hFFQ4q5pHrZBGO2fQWVGm6dZ3BhX4ldoV8LjCIzkcMMXhrhSOVjwHnqmF6xOh9EvI0jzak1w==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@mdit/plugin-uml": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@mdit/plugin-uml/-/plugin-uml-0.12.0.tgz", + "integrity": "sha512-EfVMmq0CwLJcssxhkvGS2ESenNNEMeK04j702Z9v3am1M9DdEj6zHTrHQd9tA0jNVuFY8ZlmMgDfkkG5k6Rm3Q==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "markdown-it": "^14.1.0" + }, + "peerDependenciesMeta": { + "markdown-it": { + "optional": true + } + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.1.tgz", + "integrity": "sha512-lncuC4aHicncmbORnx+dUaAgzee9cm/PbIqgWz1PpXuwc+sa1Ct83tnqUDy/GFKleLiN7ZIeytM6KJ4cAn1SxA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.1.tgz", + "integrity": "sha512-F/tkdw0WSs4ojqz5Ovrw5r9odqzFjb5LIgHdHZG65dFI1lWTWRVy32KDJLKRISHgJvqUeUhdIvy43fX41znyDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.1.tgz", + "integrity": "sha512-vk+ma8iC1ebje/ahpxpnrfVQJibTMyHdWpOGZ3JpQ7Mgn/3QNHmPq7YwjZbIE7km73dH5M1e6MRRsnEBW7v5CQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.1.tgz", + "integrity": "sha512-IgpzXKauRe1Tafcej9STjSSuG0Ghu/xGYH+qG6JwsAUxXrnkvNHcq/NL6nz1+jzvWAnQkuAJ4uIwGB48K9OCGA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.1.tgz", + "integrity": "sha512-P9bSiAUnSSM7EmyRK+e5wgpqai86QOSv8BwvkGjLwYuOpaeomiZWifEos517CwbG+aZl1T4clSE1YqqH2JRs+g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.1.tgz", + "integrity": "sha512-5RnjpACoxtS+aWOI1dURKno11d7krfpGDEn19jI8BuWmSBbUC4ytIADfROM1FZrFhQPSoP+KEa3NlEScznBTyQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.1.tgz", + "integrity": "sha512-8mwmGD668m8WaGbthrEYZ9CBmPug2QPGWxhJxh/vCgBjro5o96gL04WLlg5BA233OCWLqERy4YUzX3bJGXaJgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.1.tgz", + "integrity": "sha512-dJX9u4r4bqInMGOAQoGYdwDP8lQiisWb9et+T84l2WXk41yEej8v2iGKodmdKimT8cTAYt0jFb+UEBxnPkbXEQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.1.tgz", + "integrity": "sha512-V72cXdTl4EI0x6FNmho4D502sy7ed+LuVW6Ym8aI6DRQ9hQZdp5sj0a2usYOlqvFBNKQnLQGwmYnujo2HvjCxQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.1.tgz", + "integrity": "sha512-f+pJih7sxoKmbjghrM2RkWo2WHUW8UbfxIQiWo5yeCaCM0TveMEuAzKJte4QskBp1TIinpnRcxkquY+4WuY/tg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.1.tgz", + "integrity": "sha512-qb1hMMT3Fr/Qz1OKovCuUM11MUNLUuHeBC2DPPAWUYYUAOFWaxInaTwTQmc7Fl5La7DShTEpmYwgdt2hG+4TEg==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.1.tgz", + "integrity": "sha512-7O5u/p6oKUFYjRbZkL2FLbwsyoJAjyeXHCU3O4ndvzg2OFO2GinFPSJFGbiwFDaCFc+k7gs9CF243PwdPQFh5g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.1.tgz", + "integrity": "sha512-pDLkYITdYrH/9Cv/Vlj8HppDuLMDUBmgsM0+N+xLtFd18aXgM9Nyqupb/Uw+HeidhfYg2lD6CXvz6CjoVOaKjQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.1.tgz", + "integrity": "sha512-W2ZNI323O/8pJdBGil1oCauuCzmVd9lDmWBBqxYZcOqWD6aWqJtVBQ1dFrF4dYpZPks6F+xCZHfzG5hYlSHZ6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.1.tgz", + "integrity": "sha512-ELfEX1/+eGZYMaCIbK4jqLxO1gyTSOIlZr6pbC4SRYFaSIDVKOnZNMdoZ+ON0mrFDp4+H5MhwNC1H/AhE3zQLg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.1.tgz", + "integrity": "sha512-yjk2MAkQmoaPYCSu35RLJ62+dz358nE83VfTePJRp8CG7aMg25mEJYpXFiD+NcevhX8LxD5OP5tktPXnXN7GDw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "dev": true + }, + "node_modules/@shikijs/core": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-1.10.3.tgz", + "integrity": "sha512-D45PMaBaeDHxww+EkcDQtDAtzv00Gcsp72ukBtaLSmqRvh0WgGMq3Al0rl1QQBZfuneO75NXMIzEZGFitThWbg==", + "dev": true, + "dependencies": { + "@types/hast": "^3.0.4" + } + }, + "node_modules/@shikijs/transformers": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/@shikijs/transformers/-/transformers-1.10.3.tgz", + "integrity": "sha512-MNjsyye2WHVdxfZUSr5frS97sLGe6G1T+1P41QjyBFJehZphMcr4aBlRLmq6OSPBslYe9byQPVvt/LJCOfxw8Q==", + "dev": true, + "dependencies": { + "shiki": "1.10.3" + } + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", + "integrity": "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@stackblitz/sdk": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@stackblitz/sdk/-/sdk-1.11.0.tgz", + "integrity": "sha512-DFQGANNkEZRzFk1/rDP6TcFdM82ycHE+zfl9C/M/jXlH68jiqHWHFMQURLELoD8koxvu/eW5uhg94NSAZlYrUQ==", + "dev": true + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dev": true, + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", + "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "dev": true + }, + "node_modules/@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dev": true, + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, + "node_modules/@types/hash-sum": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/hash-sum/-/hash-sum-1.0.2.tgz", + "integrity": "sha512-UP28RddqY8xcU0SCEp9YKutQICXpaAq9N8U2klqF5hegGha7KzTOL8EdhIIV3bOSGBzjEpN9bU/d+nNZBdJYVw==", + "dev": true + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "dev": true, + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/katex": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz", + "integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==", + "dev": true + }, + "node_modules/@types/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", + "dev": true + }, + "node_modules/@types/markdown-it": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-4NpsnpYl2Gt1ljyBGrKMxFYAYvpqbnnkgP/i/g+NLpjEUa3obn1XJCur9YbEXKDAkaXqsR1LbDnGEJ0MmKFxfg==", + "dev": true, + "dependencies": { + "@types/linkify-it": "^5", + "@types/mdurl": "^2" + } + }, + "node_modules/@types/markdown-it-emoji": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/markdown-it-emoji/-/markdown-it-emoji-3.0.1.tgz", + "integrity": "sha512-cz1j8R35XivBqq9mwnsrP2fsz2yicLhB8+PDtuVkKOExwEdsVBNI+ROL3sbhtR5occRZ66vT0QnwFZCqdjf3pA==", + "dev": true, + "dependencies": { + "@types/markdown-it": "^14" + } + }, + "node_modules/@types/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", + "dev": true + }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "dev": true + }, + "node_modules/@types/node": { + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "dev": true + }, + "node_modules/@types/unist": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", + "integrity": "sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==", + "dev": true + }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.20", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.20.tgz", + "integrity": "sha512-g9gZnnXVq7gM7v3tJCWV/qw7w+KeOlSHAhgF9RytFyifW6AF61hdT2ucrYhPq9hLs5JIryeupHV3qGk95dH9ow==", + "dev": true + }, + "node_modules/@vitejs/plugin-vue": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.0.5.tgz", + "integrity": "sha512-LOjm7XeIimLBZyzinBQ6OSm3UBCNVCpLkxGC0oWmm2YPzVZoxMsdvNVimLTBzpAnR9hl/yn1SHGuRfe6/Td9rQ==", + "dev": true, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "vite": "^5.0.0", + "vue": "^3.2.25" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.4.33.tgz", + "integrity": "sha512-MoIREbkdPQlnGfSKDMgzTqzqx5nmEjIc0ydLVYlTACGBsfvOJ4tHSbZXKVF536n6fB+0eZaGEOqsGThPpdvF5A==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.24.7", + "@vue/shared": "3.4.33", + "entities": "^4.5.0", + "estree-walker": "^2.0.2", + "source-map-js": "^1.2.0" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.4.33.tgz", + "integrity": "sha512-GzB8fxEHKw0gGet5BKlpfXEqoBnzSVWwMnT+dc25wE7pFEfrU/QsvjZMP9rD4iVXHBBoemTct8mN0GJEI6ZX5A==", + "dev": true, + "dependencies": { + "@vue/compiler-core": "3.4.33", + "@vue/shared": "3.4.33" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.4.33.tgz", + "integrity": "sha512-7rk7Vbkn21xMwIUpHQR4hCVejwE6nvhBOiDgoBcR03qvGqRKA7dCBSsHZhwhYUsmjlbJ7OtD5UFIyhP6BY+c8A==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.24.7", + "@vue/compiler-core": "3.4.33", + "@vue/compiler-dom": "3.4.33", + "@vue/compiler-ssr": "3.4.33", + "@vue/shared": "3.4.33", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.10", + "postcss": "^8.4.39", + "source-map-js": "^1.2.0" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.4.33.tgz", + "integrity": "sha512-0WveC9Ai+eT/1b6LCV5IfsufBZ0HP7pSSTdDjcuW302tTEgoBw8rHVHKPbGUtzGReUFCRXbv6zQDDgucnV2WzQ==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.4.33", + "@vue/shared": "3.4.33" + } + }, + "node_modules/@vue/devtools-api": { + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.6.3.tgz", + "integrity": "sha512-0MiMsFma/HqA6g3KLKn+AGpL1kgKhFWszC9U29NfpWK5LE7bjeXxySWJrOJ77hBz+TBrBQ7o4QJqbPbqbs8rJw==", + "dev": true + }, + "node_modules/@vue/reactivity": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.4.33.tgz", + "integrity": "sha512-B24QIelahDbyHipBgbUItQblbd4w5HpG3KccL+YkGyo3maXyS253FzcTR3pSz739OTphmzlxP7JxEMWBpewilA==", + "dev": true, + "dependencies": { + "@vue/shared": "3.4.33" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.4.33.tgz", + "integrity": "sha512-6wavthExzT4iAxpe8q37/rDmf44nyOJGISJPxCi9YsQO+8w9v0gLCFLfH5TzD1V1AYrTAdiF4Y1cgUmP68jP6w==", + "dev": true, + "dependencies": { + "@vue/reactivity": "3.4.33", + "@vue/shared": "3.4.33" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.4.33.tgz", + "integrity": "sha512-iHsMCUSFJ+4z432Bn9kZzHX+zOXa6+iw36DaVRmKYZpPt9jW9riF32SxNwB124i61kp9+AZtheQ/mKoJLerAaQ==", + "dev": true, + "dependencies": { + "@vue/reactivity": "3.4.33", + "@vue/runtime-core": "3.4.33", + "@vue/shared": "3.4.33", + "csstype": "^3.1.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.4.33.tgz", + "integrity": "sha512-jTH0d6gQcaYideFP/k0WdEu8PpRS9MF8d0b6SfZzNi+ap972pZ0TNIeTaESwdOtdY0XPVj54XEJ6K0wXxir4fw==", + "dev": true, + "dependencies": { + "@vue/compiler-ssr": "3.4.33", + "@vue/shared": "3.4.33" + }, + "peerDependencies": { + "vue": "3.4.33" + } + }, + "node_modules/@vue/shared": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.4.33.tgz", + "integrity": "sha512-aoRY0jQk3A/cuvdkodTrM4NMfxco8n55eG4H7ML/CRy7OryHfiqvug4xrCBBMbbN+dvXAetDDwZW9DXWWjBntA==", + "dev": true + }, + "node_modules/@vuepress/bundler-vite": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/bundler-vite/-/bundler-vite-2.0.0-rc.14.tgz", + "integrity": "sha512-kttbowYITMCX3ztz78Qb6bMfXRv/GEpNu+nALksu7j/QJQ0gOzI2is68PatbmzZRWOufVsf1Zf0A8BwolmVcXA==", + "dev": true, + "dependencies": { + "@vitejs/plugin-vue": "^5.0.5", + "@vuepress/client": "2.0.0-rc.14", + "@vuepress/core": "2.0.0-rc.14", + "@vuepress/shared": "2.0.0-rc.14", + "@vuepress/utils": "2.0.0-rc.14", + "autoprefixer": "^10.4.19", + "connect-history-api-fallback": "^2.0.0", + "postcss": "^8.4.38", + "postcss-load-config": "^6.0.1", + "rollup": "^4.18.0", + "vite": "~5.3.1", + "vue": "^3.4.29", + "vue-router": "^4.3.3" + } + }, + "node_modules/@vuepress/cli": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/cli/-/cli-2.0.0-rc.14.tgz", + "integrity": "sha512-oYJX1nE6/ohF2tzUtpBAFxRr4MF2kdtab3+AQ897esXzrciQnE2LxPQZ8BUOn6Jb3XYW12FXDdkHrr82rN6XnQ==", + "dev": true, + "dependencies": { + "@vuepress/core": "2.0.0-rc.14", + "@vuepress/shared": "2.0.0-rc.14", + "@vuepress/utils": "2.0.0-rc.14", + "cac": "^6.7.14", + "chokidar": "^3.6.0", + "envinfo": "^7.13.0", + "esbuild": "~0.21.5" + }, + "bin": { + "vuepress-cli": "bin/vuepress.js" + } + }, + "node_modules/@vuepress/client": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/client/-/client-2.0.0-rc.14.tgz", + "integrity": "sha512-ULwxOiWoUi15HWQ6qH60gWjxSXB0797uExCUa4HgHV/8SpIqv4SHFn6jqjo7qCzOxuTqj1RT47JH3oWfUF4XPA==", + "dev": true, + "dependencies": { + "@vue/devtools-api": "^6.6.3", + "@vuepress/shared": "2.0.0-rc.14", + "vue": "^3.4.29", + "vue-router": "^4.3.3" + } + }, + "node_modules/@vuepress/core": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/core/-/core-2.0.0-rc.14.tgz", + "integrity": "sha512-Ly3fypjXGUgPzjfbXKJeyd59jxJgXkhxhWAGkH/rRyQeV8Nr7Wo1ah3H1MeGhlCRGH1T9Yd3Bz9W7QMoyWFfmg==", + "dev": true, + "dependencies": { + "@vuepress/client": "2.0.0-rc.14", + "@vuepress/markdown": "2.0.0-rc.14", + "@vuepress/shared": "2.0.0-rc.14", + "@vuepress/utils": "2.0.0-rc.14", + "vue": "^3.4.29" + } + }, + "node_modules/@vuepress/helper": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/helper/-/helper-2.0.0-rc.38.tgz", + "integrity": "sha512-IgKQCCbfX4zLkRxLwzNtTMKTZdflAlmBTUEkuD/uJrfFJjGvLShnkw2ONIlwSM6U+SWVHKfW5Ls8pndPvxpI1Q==", + "dev": true, + "dependencies": { + "@vue/shared": "^3.4.31", + "cheerio": "1.0.0-rc.12", + "fflate": "^0.8.2", + "gray-matter": "^4.0.3", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/highlighter-helper": { + "version": "2.0.0-rc.37", + "resolved": "https://registry.npmjs.org/@vuepress/highlighter-helper/-/highlighter-helper-2.0.0-rc.37.tgz", + "integrity": "sha512-l7qxuJJP0+zxDd42UctS0Oc240cCN7BvxfEx6XJfaYmn2Yncrbbk15gS9tUT3jeXB959JGm8uUhxpPP0/4w3kw==", + "dev": true, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/markdown": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/markdown/-/markdown-2.0.0-rc.14.tgz", + "integrity": "sha512-9xr693gkp71qwEbQLxpo1ybhJ+lA2k5SiuFUgqqrmR2a8CSL3gcmKEGM+y7GMnHvL63U2dYlc9pUOtJ5rG9O0Q==", + "dev": true, + "dependencies": { + "@mdit-vue/plugin-component": "^2.1.3", + "@mdit-vue/plugin-frontmatter": "^2.1.3", + "@mdit-vue/plugin-headers": "^2.1.3", + "@mdit-vue/plugin-sfc": "^2.1.3", + "@mdit-vue/plugin-title": "^2.1.3", + "@mdit-vue/plugin-toc": "^2.1.3", + "@mdit-vue/shared": "^2.1.3", + "@mdit-vue/types": "^2.1.0", + "@types/markdown-it": "^14.1.1", + "@types/markdown-it-emoji": "^3.0.1", + "@vuepress/shared": "2.0.0-rc.14", + "@vuepress/utils": "2.0.0-rc.14", + "markdown-it": "^14.1.0", + "markdown-it-anchor": "^9.0.1", + "markdown-it-emoji": "^3.0.0", + "mdurl": "^2.0.0" + } + }, + "node_modules/@vuepress/plugin-active-header-links": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-active-header-links/-/plugin-active-header-links-2.0.0-rc.38.tgz", + "integrity": "sha512-ERleWy3NBW8IWTrk8UgGMfFP1JJMi2SSGqBaQqAjkh2n2B6BSr+sY3U1yw39pnyFwdc+TwML5JomkV/W5pbTTw==", + "dev": true, + "dependencies": { + "@vueuse/core": "^10.11.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-back-to-top": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-back-to-top/-/plugin-back-to-top-2.0.0-rc.38.tgz", + "integrity": "sha512-KrEeyv2QX7YZVrvCBohPWdwFXwKFIwyb6HjEKW/2H82+XfBMg1D9b7vqCp/W4EJ1F6ERPYgJLj0sv2AockHZtg==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-blog": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-blog/-/plugin-blog-2.0.0-rc.38.tgz", + "integrity": "sha512-YcwspZIbTxdLNEjkHGAAlaLSfPubHa+j8mdI1NKWjsY1amVnDH2q3/1vIVCaP/YN9a+eHtvTXy6mikNrhnEjgw==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "chokidar": "^3.6.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-catalog": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-catalog/-/plugin-catalog-2.0.0-rc.38.tgz", + "integrity": "sha512-XvWlM3D0+zelruCuTknzBlXCcA6fF9zV7UQco/IM1YLqobwNIvJuk9w86UQSw1dh6NVrRVGX3YCz20E3bK8ByQ==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-comment": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-comment/-/plugin-comment-2.0.0-rc.38.tgz", + "integrity": "sha512-V1SqLFv3nJZN2DbMnCa9KMmrYknUwx2/KOqV3FcEXJ2uPZDY2jLnu/yolU6S3YZ6BfJPT15wPPBGlHRLDLnTOg==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "giscus": "^1.5.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "@waline/client": "^3.1.0", + "artalk": "^2.8.7", + "twikoo": "^1.5.0", + "vuepress": "2.0.0-rc.14" + }, + "peerDependenciesMeta": { + "@waline/client": { + "optional": true + }, + "artalk": { + "optional": true + }, + "twikoo": { + "optional": true + } + } + }, + "node_modules/@vuepress/plugin-copy-code": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-copy-code/-/plugin-copy-code-2.0.0-rc.38.tgz", + "integrity": "sha512-SKO88dEwkCn+2YRNGZJHId8sulas4B8tD8suzSwNyTi+9+T54hrhQpc4cN1GjfbxOPFcINAQsndZFKMX6Wg6tg==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-copyright": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-copyright/-/plugin-copyright-2.0.0-rc.38.tgz", + "integrity": "sha512-ILkUG2yscwajwfa/Ox1QLZJcn76XGRFaRsRm9O/MzkHtZ/qRJ1QZa0ccqpYPKmNWAPw7isPfi95qeAg0nl34KQ==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-git": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-git/-/plugin-git-2.0.0-rc.38.tgz", + "integrity": "sha512-dRJiZ5PVuhhyu+R2BZOlyeqgxVikUUh2Vf6RNVN2DNWv4VHdYybFQuQ+kYDpldYyzoP8932aFRV0d2ocpvxEug==", + "dev": true, + "dependencies": { + "execa": "^9.3.0" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-links-check": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-links-check/-/plugin-links-check-2.0.0-rc.38.tgz", + "integrity": "sha512-UAf7WpfIdMYD14H3N7oXhriOHmWiErWpNGaRGauZvTXPZV3VE8sSylZ7ezsYjepvWstFGqfN0AmBvl46S++AJg==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-notice": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-notice/-/plugin-notice-2.0.0-rc.38.tgz", + "integrity": "sha512-HYS2fTXU8/Nw6S1U3opWMkbGe70I+e/ADyCbcix0Zhz+/5RNoFIuoAT9TmAC2vqO0Bl/LwaLFZItj3frK0m8IQ==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-nprogress": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-nprogress/-/plugin-nprogress-2.0.0-rc.38.tgz", + "integrity": "sha512-KtTvwOA25n8KtXQ1adL3Pa0vf2OdosAxG4UMYGbULe1ScCH9ER86B+cD3vG+pAXLSolNlKmg+VsGjQcO+vF4jQ==", + "dev": true, + "dependencies": { + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-photo-swipe": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-photo-swipe/-/plugin-photo-swipe-2.0.0-rc.38.tgz", + "integrity": "sha512-S1PtFr6UZ7zTp+J6TorOzuXC2lpcQJ/G7iTS24px2swyzELfpBjhf3gdOKl9wjsnqJ/4DdyUkoHUX0dA7piAkw==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "photoswipe": "^5.4.4", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-reading-time": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-reading-time/-/plugin-reading-time-2.0.0-rc.38.tgz", + "integrity": "sha512-ocX2nmvUNbXhuCizTIVZIR2K1pmYA1vA741IH5BPhvD8y5JYc9mZsudJZLPRUgYEi/Yx2o6XMnwqAKrwCqypig==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-rtl": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-rtl/-/plugin-rtl-2.0.0-rc.38.tgz", + "integrity": "sha512-Jfw0iQXRwddi8GiwBB78+Sdonp5VUZGE2md1IUu9xQuMJ4oW6Fgp1duIp2rWEGiiezwz6CqQb2a/ph3s8NYgPw==", + "dev": true, + "dependencies": { + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-sass-palette": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-sass-palette/-/plugin-sass-palette-2.0.0-rc.38.tgz", + "integrity": "sha512-lYVtH02y1bG62sc0r2IiSxNqm+5bQYr75GhEqEnFYh9ZmXLLKdCIw84T/JcK4GLi3MOn5rCPO3o4O6DaPM817g==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "chokidar": "^3.6.0", + "sass": "^1.77.8" + }, + "peerDependencies": { + "sass-loader": "^14.0.0", + "vuepress": "2.0.0-rc.14" + }, + "peerDependenciesMeta": { + "sass-loader": { + "optional": true + } + } + }, + "node_modules/@vuepress/plugin-search": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-search/-/plugin-search-2.0.0-rc.38.tgz", + "integrity": "sha512-5FTyewhnE9me8EzTXVPJYvnDyo90gw0shkXDfqmImqygk3LFcpKSxJE7imSIPjFBl+Vl9zop+EK2YJF/l4KvlQ==", + "dev": true, + "optional": true, + "peer": true, + "dependencies": { + "chokidar": "^3.6.0", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-seo": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-seo/-/plugin-seo-2.0.0-rc.38.tgz", + "integrity": "sha512-H/f2AtEYohFy0QJv/mbwmdDj0NgY5mpbKY1GtPW+LBn5fymWJ37TZ+gz+/+Htq4BdEbWQmcIj2XteTxdzeyAvQ==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-shiki": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-shiki/-/plugin-shiki-2.0.0-rc.38.tgz", + "integrity": "sha512-e+4OiDxry48bGskgvpxItsNxLOjAckMjASNcH9xGaXeUH3LiwTaCDIBbb89GeIh1LJ4Xnjw1HMlkpomgQmrI8A==", + "dev": true, + "dependencies": { + "@shikijs/transformers": "^1.10.3", + "@vuepress/helper": "2.0.0-rc.38", + "@vuepress/highlighter-helper": "2.0.0-rc.37", + "nanoid": "^5.0.7", + "shiki": "^1.10.3" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-shiki/node_modules/nanoid": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz", + "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^18 || >=20" + } + }, + "node_modules/@vuepress/plugin-sitemap": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-sitemap/-/plugin-sitemap-2.0.0-rc.38.tgz", + "integrity": "sha512-R5arITfgVxvdpsapUG48vWR0/loy70MGYC97XU5m9BbNh/Wd7y+QXn6OOAXTAT4U4Vl6c18zMeq3kVyrhj0dKQ==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "sitemap": "^8.0.0" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-theme-data": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-theme-data/-/plugin-theme-data-2.0.0-rc.38.tgz", + "integrity": "sha512-YeqhtIzBbdB9OORY2M/N+c8O0HqqfEj/6H8waGXfBxHnb/M+kTJ2PnmCjiqrjiTG1JuhuZOpxIjG/rio2VV3+A==", + "dev": true, + "dependencies": { + "@vue/devtools-api": "^6.6.3", + "vue": "^3.4.31" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/plugin-watermark": { + "version": "2.0.0-rc.38", + "resolved": "https://registry.npmjs.org/@vuepress/plugin-watermark/-/plugin-watermark-2.0.0-rc.38.tgz", + "integrity": "sha512-5lFQ2Cr5PXS57IzGOC0X4h/4XPmC9FSyEweeR3cGMhLqJ1CST0TplUzRftVNsqoXibH/2/UKBQch01tQxzrCAg==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "vue": "^3.4.31", + "watermark-js-plus": "^1.5.2" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/@vuepress/shared": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/shared/-/shared-2.0.0-rc.14.tgz", + "integrity": "sha512-VDDnPpz4x1Q07richcVRGbc4qc2RG/6bKoEYSImofTFzvdmHer538ouv8kD2SNU10UrSOpxxUiphnhlhNIe03A==", + "dev": true, + "dependencies": { + "@mdit-vue/types": "^2.1.0" + } + }, + "node_modules/@vuepress/utils": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/@vuepress/utils/-/utils-2.0.0-rc.14.tgz", + "integrity": "sha512-1h/5qcKBeIhIg6SZM2IoZVOaIdFSeQ1CdEWadqQWy1uwupEeVrU3QPkjFyn0vUt0O/EuuVqQcLLC8OuS/wldNw==", + "dev": true, + "dependencies": { + "@types/debug": "^4.1.12", + "@types/fs-extra": "^11.0.4", + "@types/hash-sum": "^1.0.2", + "@vuepress/shared": "2.0.0-rc.14", + "debug": "^4.3.5", + "fs-extra": "^11.2.0", + "globby": "^14.0.1", + "hash-sum": "^2.0.0", + "ora": "^8.0.1", + "picocolors": "^1.0.1", + "upath": "^2.0.1" + } + }, + "node_modules/@vueuse/core": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-10.11.0.tgz", + "integrity": "sha512-x3sD4Mkm7PJ+pcq3HX8PLPBadXCAlSDR/waK87dz0gQE+qJnaaFhc/dZVfJz+IUYzTMVGum2QlR7ImiJQN4s6g==", + "dev": true, + "dependencies": { + "@types/web-bluetooth": "^0.0.20", + "@vueuse/metadata": "10.11.0", + "@vueuse/shared": "10.11.0", + "vue-demi": ">=0.14.8" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/core/node_modules/vue-demi": { + "version": "0.14.8", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.8.tgz", + "integrity": "sha512-Uuqnk9YE9SsWeReYqK2alDI5YzciATE0r2SkA6iMAtuXvNTMNACJLJEXNXaEy94ECuBe4Sk6RzRU80kjdbIo1Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/metadata": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-10.11.0.tgz", + "integrity": "sha512-kQX7l6l8dVWNqlqyN3ePW3KmjCQO3ZMgXuBMddIu83CmucrsBfXlH+JoviYyRBws/yLTQO8g3Pbw+bdIoVm4oQ==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-10.11.0.tgz", + "integrity": "sha512-fyNoIXEq3PfX1L3NkNhtVQUSRtqYwJtJg+Bp9rIzculIZWHTkKSysujrOk2J+NrRulLTQH9+3gGSfYLWSEWU1A==", + "dev": true, + "dependencies": { + "vue-demi": ">=0.14.8" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared/node_modules/vue-demi": { + "version": "0.14.8", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.8.tgz", + "integrity": "sha512-Uuqnk9YE9SsWeReYqK2alDI5YzciATE0r2SkA6iMAtuXvNTMNACJLJEXNXaEy94ECuBe4Sk6RzRU80kjdbIo1Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/autoprefixer": { + "version": "10.4.19", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.19.tgz", + "integrity": "sha512-BaENR2+zBZ8xXhM4pUaKUxlVdxZ0EZhjvbopwnXmxRUfqDmwSpC2lAi/QXvx7NRdPCo1WKEcEF6mV64si1z4Ew==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "browserslist": "^4.23.0", + "caniuse-lite": "^1.0.30001599", + "fraction.js": "^4.3.7", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.0", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/balloon-css": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/balloon-css/-/balloon-css-1.2.0.tgz", + "integrity": "sha512-urXwkHgwp6GsXVF+it01485Z2Cj4pnW02ICnM0TemOlkKmCNnDLmyy+ZZiRXBpwldUXO+aRNr7Hdia4CBvXJ5A==", + "dev": true + }, + "node_modules/bcrypt-ts": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-ts/-/bcrypt-ts-5.0.2.tgz", + "integrity": "sha512-gDwQ5784AkkfhHACh3jGcg1hUubyZyeq9AtVd5gXkcyHGVOC+mORjRIHSj+fHfqwY5vxwyBLXQpcfk8MpK0ROg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.23.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.2.tgz", + "integrity": "sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001640", + "electron-to-chromium": "^1.4.820", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.1.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001642", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz", + "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "dev": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dev": true, + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/cli-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", + "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", + "dev": true, + "dependencies": { + "restore-cursor": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/create-codepen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/create-codepen/-/create-codepen-2.0.0.tgz", + "integrity": "sha512-ehJ0Zw5RSV2G4+/azUb7vEZWRSA/K9cW7HDock1Y9ViDexkgSJUZJRcObdw/YAWeXKjreEQV9l/igNSsJ1yw5A==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "dev": true + }, + "node_modules/dayjs": { + "version": "1.11.12", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.12.tgz", + "integrity": "sha512-Rt2g+nTbLlDWZTwwrIXjy9MeiZmSDI375FvZs72ngxx8PDC6YXOeR3q5LAuPzjZQxhiWdRKac7RKV+YyQYfYIg==", + "dev": true + }, + "node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dijkstrajs": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/dijkstrajs/-/dijkstrajs-1.0.3.tgz", + "integrity": "sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==", + "dev": true + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dev": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dev": true, + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dev": true, + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.830", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.830.tgz", + "integrity": "sha512-TrPKKH20HeN0J1LHzsYLs2qwXrp8TF4nHdu4sq61ozGbzMpWhI7iIOPYPPkxeq1azMT9PZ8enPFcftbs/Npcjg==", + "dev": true + }, + "node_modules/emoji-regex": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", + "integrity": "sha512-QpLs9D9v9kArv4lfDEgg1X/gN5XLnf/A6l9cs8SPZLRZR3ZkY9+kwIQTxm+fsSej5UMYGE8fdoaZVIBlqG0XTw==", + "dev": true + }, + "node_modules/encode-utf8": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/encode-utf8/-/encode-utf8-1.0.3.tgz", + "integrity": "sha512-ucAnuBEhUK4boH2HjVYG5Q2mQyPorvv0u/ocS+zhdw0S8AlHYY+GOFhP1Gio5z4icpP2ivFSvhtFjQi8+T9ppw==", + "dev": true + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/envinfo": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.13.0.tgz", + "integrity": "sha512-cvcaMr7KqXVh4nyzGTVqTum+gAiL265x5jUWQIDLq//zOGbW+gSW/C+OWLleY/rs9Qole6AZLMXPbtIFQbqu+Q==", + "dev": true, + "bin": { + "envinfo": "dist/cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, + "node_modules/execa": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-9.3.0.tgz", + "integrity": "sha512-l6JFbqnHEadBoVAVpN5dl2yCyfX28WoBAGaoQcNmLLSedOxTxcn2Qa83s8I/PA5i56vWru2OHOtrwF7Om2vqlg==", + "dev": true, + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "cross-spawn": "^7.0.3", + "figures": "^6.1.0", + "get-stream": "^9.0.0", + "human-signals": "^7.0.0", + "is-plain-obj": "^4.1.0", + "is-stream": "^4.0.1", + "npm-run-path": "^5.2.0", + "pretty-ms": "^9.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^4.0.0", + "yoctocolors": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.5.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-glob": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "dev": true + }, + "node_modules/figures": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", + "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", + "dev": true, + "dependencies": { + "is-unicode-supported": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fraction.js": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", + "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==", + "dev": true, + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.2.0.tgz", + "integrity": "sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "dev": true, + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/giscus": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/giscus/-/giscus-1.5.0.tgz", + "integrity": "sha512-t3LL0qbSO3JXq3uyQeKpF5CegstGfKX/0gI6eDe1cmnI7D56R7j52yLdzw4pdKrg3VnufwCgCM3FDz7G1Qr6lg==", + "dev": true, + "dependencies": { + "lit": "^3.1.2" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-14.0.2.tgz", + "integrity": "sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==", + "dev": true, + "dependencies": { + "@sindresorhus/merge-streams": "^2.1.0", + "fast-glob": "^3.3.2", + "ignore": "^5.2.4", + "path-type": "^5.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true + }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "dev": true, + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/hash-sum": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hash-sum/-/hash-sum-2.0.0.tgz", + "integrity": "sha512-WdZTbAByD+pHfl/g9QSsBIIwy8IT+EsPiKDs0KNX+zSHhdDLFKdZu0BQHljvO+0QI/BasbMSUa8wYNCZTvhslg==", + "dev": true + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/human-signals": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-7.0.0.tgz", + "integrity": "sha512-74kytxOUSvNbjrT9KisAbaTZ/eJwD/LrbM/kh5j0IhPuJzwuA19dWvniFGwBzN9rVjg+O/e+F310PjObDXS+9Q==", + "dev": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/ignore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/immutable": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", + "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-interactive": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.0.0.tgz", + "integrity": "sha512-FRdAyx5lusK1iHG0TWpVtk9+1i+GjrzRffhDg4ovQ7mcidMQ6mj+MhKPmvh7Xwyv5gIS06ns49CA7Sqg7lC22Q==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lilconfig": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.2.tgz", + "integrity": "sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "dev": true, + "dependencies": { + "uc.micro": "^2.0.0" + } + }, + "node_modules/lit": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/lit/-/lit-3.1.4.tgz", + "integrity": "sha512-q6qKnKXHy2g1kjBaNfcoLlgbI3+aSOZ9Q4tiGa9bGYXq5RBXxkVTqTIVmP2VWMp29L4GyvCFm8ZQ2o56eUAMyA==", + "dev": true, + "dependencies": { + "@lit/reactive-element": "^2.0.4", + "lit-element": "^4.0.4", + "lit-html": "^3.1.2" + } + }, + "node_modules/lit-element": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-4.0.6.tgz", + "integrity": "sha512-U4sdJ3CSQip7sLGZ/uJskO5hGiqtlpxndsLr6mt3IQIjheg93UKYeGQjWMRql1s/cXNOaRrCzC2FQwjIwSUqkg==", + "dev": true, + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.2.0", + "@lit/reactive-element": "^2.0.4", + "lit-html": "^3.1.2" + } + }, + "node_modules/lit-html": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-3.1.4.tgz", + "integrity": "sha512-yKKO2uVv7zYFHlWMfZmqc+4hkmSbFp8jgjdZY9vvR9jr4J8fH6FUMXhr+ljfELgmjpvlF7Z1SJ5n5/Jeqtc9YA==", + "dev": true, + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/log-symbols": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", + "integrity": "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==", + "dev": true, + "dependencies": { + "chalk": "^5.3.0", + "is-unicode-supported": "^1.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols/node_modules/is-unicode-supported": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/magic-string": { + "version": "0.30.10", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.10.tgz", + "integrity": "sha512-iIRwTIf0QKV3UAnYK4PU8uiEc4SRh5jX0mwpIwETPpHdhVM4f53RSwS/vXvN1JhGX+Cs7B8qIq3d6AH49O5fAQ==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + } + }, + "node_modules/markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdown-it-anchor": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-9.0.1.tgz", + "integrity": "sha512-cBt7aAzmkfX8X7FqAe8EBryiKmToXgMQEEMqkXzWCm0toDtfDYIGboKeTKd8cpNJArJtutrf+977wFJTsvNGmQ==", + "dev": true, + "peerDependencies": { + "@types/markdown-it": "*", + "markdown-it": "*" + } + }, + "node_modules/markdown-it-emoji": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/markdown-it-emoji/-/markdown-it-emoji-3.0.0.tgz", + "integrity": "sha512-+rUD93bXHubA4arpEZO3q80so0qgoFJEKRkRbjKX8RTdca89v2kfyF+xR3i2sQTwql9tpPZPOQN5B+PunspXRg==", + "dev": true + }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "dev": true + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "dev": true, + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.17", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.17.tgz", + "integrity": "sha512-Ww6ZlOiEQfPfXM45v17oabk77Z7mg5bOt7AjDyzy7RjK9OrLrLC8dyZQoAPEOtFX9SaNf1Tdvr5gRJWdTJj7GA==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-8.0.1.tgz", + "integrity": "sha512-ANIvzobt1rls2BDny5fWZ3ZVKyD6nscLvfFRpQgfWsythlcsVUC9kL0zq6j2Z5z9wwp1kd7wpsD/T9qNPVLCaQ==", + "dev": true, + "dependencies": { + "chalk": "^5.3.0", + "cli-cursor": "^4.0.0", + "cli-spinners": "^2.9.2", + "is-interactive": "^2.0.0", + "is-unicode-supported": "^2.0.0", + "log-symbols": "^6.0.0", + "stdin-discarder": "^0.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-ms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", + "integrity": "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dev": true, + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dev": true, + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-type": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-5.0.0.tgz", + "integrity": "sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/photoswipe": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/photoswipe/-/photoswipe-5.4.4.tgz", + "integrity": "sha512-WNFHoKrkZNnvFFhbHL93WDkW3ifwVOXSW3w1UuZZelSmgXpIGiZSNlZJq37rR8YejqME2rHs9EhH9ZvlvFH2NA==", + "dev": true, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/picocolors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pngjs": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", + "integrity": "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==", + "dev": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/postcss": { + "version": "8.4.39", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.39.tgz", + "integrity": "sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.0.1", + "source-map-js": "^1.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "node_modules/pretty-ms": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.0.0.tgz", + "integrity": "sha512-E9e9HJ9R9NasGOgPaPE8VMeiPKAyWR5jcFpNnwIejslIhWqdqOrb2wShBsncMPUb+BcCd2OPYfh7p2W6oemTng==", + "dev": true, + "dependencies": { + "parse-ms": "^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qrcode": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.3.tgz", + "integrity": "sha512-puyri6ApkEHYiVl4CFzo1tDkAZ+ATcnbJrJ6RiBM1Fhctdn/ix9MTE3hRph33omisEbC/2fcfemsseiKgBPKZg==", + "dev": true, + "dependencies": { + "dijkstrajs": "^1.0.1", + "encode-utf8": "^1.0.3", + "pngjs": "^5.0.0", + "yargs": "^15.3.1" + }, + "bin": { + "qrcode": "bin/qrcode" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "node_modules/restore-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", + "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", + "dev": true, + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.18.1.tgz", + "integrity": "sha512-Elx2UT8lzxxOXMpy5HWQGZqkrQOtrVDDa/bm9l10+U4rQnVzbL/LgZ4NOM1MPIDyHk69W4InuYDF5dzRh4Kw1A==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.5" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.18.1", + "@rollup/rollup-android-arm64": "4.18.1", + "@rollup/rollup-darwin-arm64": "4.18.1", + "@rollup/rollup-darwin-x64": "4.18.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.18.1", + "@rollup/rollup-linux-arm-musleabihf": "4.18.1", + "@rollup/rollup-linux-arm64-gnu": "4.18.1", + "@rollup/rollup-linux-arm64-musl": "4.18.1", + "@rollup/rollup-linux-powerpc64le-gnu": "4.18.1", + "@rollup/rollup-linux-riscv64-gnu": "4.18.1", + "@rollup/rollup-linux-s390x-gnu": "4.18.1", + "@rollup/rollup-linux-x64-gnu": "4.18.1", + "@rollup/rollup-linux-x64-musl": "4.18.1", + "@rollup/rollup-win32-arm64-msvc": "4.18.1", + "@rollup/rollup-win32-ia32-msvc": "4.18.1", + "@rollup/rollup-win32-x64-msvc": "4.18.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/sass": { + "version": "1.77.8", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.77.8.tgz", + "integrity": "sha512-4UHg6prsrycW20fqLGPShtEvo/WyHRVRHwOP4DzkUrObWoWI05QBSfzU71TVB7PFaL104TwNaHpjlWXAZbQiNQ==", + "dev": true, + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "dev": true + }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/shiki": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-1.10.3.tgz", + "integrity": "sha512-eneCLncGuvPdTutJuLyUGS8QNPAVFO5Trvld2wgEq1e002mwctAhJKeMGWtWVXOIEzmlcLRqcgPSorR6AVzOmQ==", + "dev": true, + "dependencies": { + "@shikijs/core": "1.10.3", + "@types/hast": "^3.0.4" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/sitemap": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-8.0.0.tgz", + "integrity": "sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==", + "dev": true, + "dependencies": { + "@types/node": "^17.0.5", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.2.4" + }, + "bin": { + "sitemap": "dist/cli.js" + }, + "engines": { + "node": ">=14.0.0", + "npm": ">=6.0.0" + } + }, + "node_modules/sitemap/node_modules/@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "dev": true + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "dev": true, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/slimsearch": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/slimsearch/-/slimsearch-2.1.1.tgz", + "integrity": "sha512-l1utJWal8F/RIheYk88DE2+enI12nIrn5SHt4ih/CNAH81PzkTv2GVBODlLynDJb7xan5hjd8XTL5f0L4cxLQA==", + "dev": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true + }, + "node_modules/stdin-discarder": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.2.2.tgz", + "integrity": "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", + "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "dev": true + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, + "node_modules/unicorn-magic": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz", + "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/upath": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", + "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", + "dev": true, + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", + "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.1.2", + "picocolors": "^1.0.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.4.tgz", + "integrity": "sha512-Cw+7zL3ZG9/NZBB8C+8QbQZmR54GwqIz+WMI4b3JgdYJvX+ny9AjJXqkGQlDXSXRP9rP0B4tbciRMOVEKulVOA==", + "dev": true, + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.39", + "rollup": "^4.13.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vue": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.4.33.tgz", + "integrity": "sha512-VdMCWQOummbhctl4QFMcW6eNtXHsFyDlX60O/tsSQuCcuDOnJ1qPOhhVla65Niece7xq/P2zyZReIO5mP+LGTQ==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.4.33", + "@vue/compiler-sfc": "3.4.33", + "@vue/runtime-dom": "3.4.33", + "@vue/server-renderer": "3.4.33", + "@vue/shared": "3.4.33" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/vue-router": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.4.0.tgz", + "integrity": "sha512-HB+t2p611aIZraV2aPSRNXf0Z/oLZFrlygJm+sZbdJaW6lcFqEDQwnzUBXn+DApw+/QzDU/I9TeWx9izEjTmsA==", + "dev": true, + "dependencies": { + "@vue/devtools-api": "^6.5.1" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "vue": "^3.2.0" + } + }, + "node_modules/vuepress": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/vuepress/-/vuepress-2.0.0-rc.14.tgz", + "integrity": "sha512-t902FYKFF2MavNQjm/I4gN8etl6iX4PETutu4c1Pt7qQjXF6Hp2eurZaW32O5/TaYWsbVG757FwKodRLj9GDng==", + "dev": true, + "dependencies": { + "@vuepress/cli": "2.0.0-rc.14", + "@vuepress/client": "2.0.0-rc.14", + "@vuepress/core": "2.0.0-rc.14", + "@vuepress/markdown": "2.0.0-rc.14", + "@vuepress/shared": "2.0.0-rc.14", + "@vuepress/utils": "2.0.0-rc.14", + "vue": "^3.4.29" + }, + "bin": { + "vuepress": "bin/vuepress.js", + "vuepress-vite": "bin/vuepress-vite.js", + "vuepress-webpack": "bin/vuepress-webpack.js" + }, + "engines": { + "node": ">=18.16.0" + }, + "peerDependencies": { + "@vuepress/bundler-vite": "2.0.0-rc.14", + "@vuepress/bundler-webpack": "2.0.0-rc.14", + "vue": "^3.4.0" + }, + "peerDependenciesMeta": { + "@vuepress/bundler-vite": { + "optional": true + }, + "@vuepress/bundler-webpack": { + "optional": true + } + } + }, + "node_modules/vuepress-plugin-components": { + "version": "2.0.0-rc.51", + "resolved": "https://registry.npmjs.org/vuepress-plugin-components/-/vuepress-plugin-components-2.0.0-rc.51.tgz", + "integrity": "sha512-oPlym6g6hAqX9PzV40jrCqrST5XNz2QLYUP+OtvT6iStJkmyjirVOJ/0eVLXRo8lVfqntcBEKebZ5dhz5qYB9w==", + "dev": true, + "dependencies": { + "@stackblitz/sdk": "^1.11.0", + "@vuepress/helper": "2.0.0-rc.38", + "@vuepress/plugin-sass-palette": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "balloon-css": "^1.2.0", + "create-codepen": "^2.0.0", + "qrcode": "^1.5.3", + "vue": "^3.4.31", + "vuepress-shared": "2.0.0-rc.51" + }, + "engines": { + "node": ">=18.19.0", + "npm": ">=8", + "pnpm": ">=7", + "yarn": ">=2" + }, + "peerDependencies": { + "artplayer": "^5.0.0", + "dashjs": "4.7.4", + "hls.js": "^1.4.12", + "mpegts.js": "^1.7.3", + "sass-loader": "^14.0.0", + "vidstack": "^1.11.21", + "vuepress": "2.0.0-rc.14" + }, + "peerDependenciesMeta": { + "artplayer": { + "optional": true + }, + "dashjs": { + "optional": true + }, + "hls.js": { + "optional": true + }, + "mpegts.js": { + "optional": true + }, + "sass-loader": { + "optional": true + }, + "vidstack": { + "optional": true + } + } + }, + "node_modules/vuepress-plugin-md-enhance": { + "version": "2.0.0-rc.51", + "resolved": "https://registry.npmjs.org/vuepress-plugin-md-enhance/-/vuepress-plugin-md-enhance-2.0.0-rc.51.tgz", + "integrity": "sha512-m8J9DUleFvwfNujgRgRlkwddPTCyumtlPtOXgRpnnTn1uiXVtJoq4PwFMY7PiNMoNr0Q/6a6fuDjEJfNgG/l1w==", + "dev": true, + "dependencies": { + "@mdit/plugin-alert": "^0.12.0", + "@mdit/plugin-align": "^0.12.0", + "@mdit/plugin-attrs": "^0.12.0", + "@mdit/plugin-container": "^0.12.0", + "@mdit/plugin-demo": "^0.12.0", + "@mdit/plugin-figure": "^0.12.0", + "@mdit/plugin-footnote": "^0.12.0", + "@mdit/plugin-img-lazyload": "^0.12.0", + "@mdit/plugin-img-mark": "^0.12.0", + "@mdit/plugin-img-size": "^0.12.0", + "@mdit/plugin-include": "^0.12.0", + "@mdit/plugin-katex-slim": "^0.12.0", + "@mdit/plugin-mark": "^0.12.0", + "@mdit/plugin-mathjax-slim": "^0.12.0", + "@mdit/plugin-plantuml": "^0.12.0", + "@mdit/plugin-spoiler": "^0.12.0", + "@mdit/plugin-stylize": "^0.12.0", + "@mdit/plugin-sub": "^0.12.0", + "@mdit/plugin-sup": "^0.12.0", + "@mdit/plugin-tab": "^0.12.0", + "@mdit/plugin-tasklist": "^0.12.0", + "@mdit/plugin-tex": "^0.12.0", + "@mdit/plugin-uml": "^0.12.0", + "@types/markdown-it": "^14.1.1", + "@vuepress/helper": "2.0.0-rc.38", + "@vuepress/plugin-sass-palette": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "balloon-css": "^1.2.0", + "js-yaml": "^4.1.0", + "vue": "^3.4.31", + "vuepress-shared": "2.0.0-rc.51" + }, + "engines": { + "node": ">=18.19.0", + "npm": ">=8", + "pnpm": ">=7", + "yarn": ">=2" + }, + "peerDependencies": { + "@types/reveal.js": "^5.0.0", + "@vue/repl": "^4.1.1", + "chart.js": "^4.0.0", + "echarts": "^5.0.0", + "flowchart.ts": "^2.0.0 || ^3.0.0", + "katex": "^0.16.0", + "kotlin-playground": "^1.23.0", + "markmap-lib": "^0.17.0", + "markmap-toolbar": "^0.17.0", + "markmap-view": "^0.17.0", + "mathjax-full": "^3.2.2", + "mermaid": "^10.8.0", + "reveal.js": "^5.0.0", + "sandpack-vue3": "^3.0.0", + "sass-loader": "^14.0.0", + "vuepress": "2.0.0-rc.14" + }, + "peerDependenciesMeta": { + "@types/reveal.js": { + "optional": true + }, + "@vue/repl": { + "optional": true + }, + "chart.js": { + "optional": true + }, + "echarts": { + "optional": true + }, + "flowchart.ts": { + "optional": true + }, + "katex": { + "optional": true + }, + "kotlin-playground": { + "optional": true + }, + "markmap-lib": { + "optional": true + }, + "markmap-toolbar": { + "optional": true + }, + "markmap-view": { + "optional": true + }, + "mathjax-full": { + "optional": true + }, + "mermaid": { + "optional": true + }, + "reveal.js": { + "optional": true + }, + "sandpack-vue3": { + "optional": true + }, + "sass-loader": { + "optional": true + } + } + }, + "node_modules/vuepress-plugin-search-pro": { + "version": "2.0.0-rc.51", + "resolved": "https://registry.npmjs.org/vuepress-plugin-search-pro/-/vuepress-plugin-search-pro-2.0.0-rc.51.tgz", + "integrity": "sha512-k4w1CyX3ZSQA9oxH2bwYL25R+qg6Lfvt8mYz88zwTY1a8i9Un8ngkxwQagmwD8cebyQ/YzTRoQfLwBRO9WU6tw==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vuepress/plugin-sass-palette": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "cheerio": "1.0.0-rc.12", + "chokidar": "^3.6.0", + "slimsearch": "^2.1.1", + "vue": "^3.4.31", + "vuepress-shared": "2.0.0-rc.51" + }, + "engines": { + "node": ">=18.19.0", + "npm": ">=8", + "pnpm": ">=7", + "yarn": ">=2" + }, + "peerDependencies": { + "sass-loader": "^14.0.0", + "vuepress": "2.0.0-rc.14" + }, + "peerDependenciesMeta": { + "sass-loader": { + "optional": true + } + } + }, + "node_modules/vuepress-shared": { + "version": "2.0.0-rc.51", + "resolved": "https://registry.npmjs.org/vuepress-shared/-/vuepress-shared-2.0.0-rc.51.tgz", + "integrity": "sha512-9E/7nV1qe9A2gydeYClkmPH6McXOrI26rdFXybrMEbBHUhzmqsBN7GaM3YxNE6qSqGzpEBOOuurBllJFKwd66A==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "cheerio": "1.0.0-rc.12", + "dayjs": "^1.11.11", + "execa": "^9.3.0", + "fflate": "^0.8.2", + "gray-matter": "^4.0.3", + "semver": "^7.6.2", + "vue": "^3.4.31" + }, + "engines": { + "node": ">=18.19.0", + "npm": ">=8", + "pnpm": ">=7", + "yarn": ">=2" + }, + "peerDependencies": { + "vuepress": "2.0.0-rc.14" + } + }, + "node_modules/vuepress-theme-hope": { + "version": "2.0.0-rc.51", + "resolved": "https://registry.npmjs.org/vuepress-theme-hope/-/vuepress-theme-hope-2.0.0-rc.51.tgz", + "integrity": "sha512-Q7AIm8pp0mWtWWiJmLSpWo7yY6a8e4JJx5V0sktD8dDsiscBUB4ct3jimJQWe7X6QJx9uOFEVaulHAtSm3I4lw==", + "dev": true, + "dependencies": { + "@vuepress/helper": "2.0.0-rc.38", + "@vuepress/plugin-active-header-links": "2.0.0-rc.38", + "@vuepress/plugin-back-to-top": "2.0.0-rc.38", + "@vuepress/plugin-blog": "2.0.0-rc.38", + "@vuepress/plugin-catalog": "2.0.0-rc.38", + "@vuepress/plugin-comment": "2.0.0-rc.38", + "@vuepress/plugin-copy-code": "2.0.0-rc.38", + "@vuepress/plugin-copyright": "2.0.0-rc.38", + "@vuepress/plugin-git": "2.0.0-rc.38", + "@vuepress/plugin-links-check": "2.0.0-rc.38", + "@vuepress/plugin-notice": "2.0.0-rc.38", + "@vuepress/plugin-nprogress": "2.0.0-rc.38", + "@vuepress/plugin-photo-swipe": "2.0.0-rc.38", + "@vuepress/plugin-reading-time": "2.0.0-rc.38", + "@vuepress/plugin-rtl": "2.0.0-rc.38", + "@vuepress/plugin-sass-palette": "2.0.0-rc.38", + "@vuepress/plugin-seo": "2.0.0-rc.38", + "@vuepress/plugin-shiki": "2.0.0-rc.38", + "@vuepress/plugin-sitemap": "2.0.0-rc.38", + "@vuepress/plugin-theme-data": "2.0.0-rc.38", + "@vuepress/plugin-watermark": "2.0.0-rc.38", + "@vueuse/core": "^10.11.0", + "balloon-css": "^1.2.0", + "bcrypt-ts": "^5.0.2", + "cheerio": "1.0.0-rc.12", + "chokidar": "^3.6.0", + "gray-matter": "^4.0.3", + "vue": "^3.4.31", + "vuepress-plugin-components": "2.0.0-rc.51", + "vuepress-plugin-md-enhance": "2.0.0-rc.51", + "vuepress-shared": "2.0.0-rc.51" + }, + "engines": { + "node": ">=18.19.0", + "npm": ">=8", + "pnpm": ">=7", + "yarn": ">=2" + }, + "peerDependencies": { + "@vuepress/plugin-docsearch": "2.0.0-rc.38", + "@vuepress/plugin-feed": "2.0.0-rc.38", + "@vuepress/plugin-prismjs": "2.0.0-rc.37", + "@vuepress/plugin-pwa": "2.0.0-rc.38", + "@vuepress/plugin-redirect": "2.0.0-rc.38", + "@vuepress/plugin-search": "2.0.0-rc.38", + "nodejs-jieba": "^0.1.2", + "sass-loader": "^14.0.0", + "vuepress": "2.0.0-rc.14", + "vuepress-plugin-search-pro": "2.0.0-rc.51" + }, + "peerDependenciesMeta": { + "@vuepress/plugin-docsearch": { + "optional": true + }, + "@vuepress/plugin-feed": { + "optional": true + }, + "@vuepress/plugin-prismjs": { + "optional": true + }, + "@vuepress/plugin-pwa": { + "optional": true + }, + "@vuepress/plugin-redirect": { + "optional": true + }, + "@vuepress/plugin-search": { + "optional": true + }, + "nodejs-jieba": { + "optional": true + }, + "sass-loader": { + "optional": true + }, + "vuepress-plugin-search-pro": { + "optional": true + } + } + }, + "node_modules/watermark-js-plus": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/watermark-js-plus/-/watermark-js-plus-1.5.2.tgz", + "integrity": "sha512-iqgSeAfwnCKNpClmyjl7rhj0SEbt8j+MqZc6C3YKY5xjMdxlRMIOcnYdBYBiznzILVyJ6YbwxD5OMajK1D+uCA==", + "dev": true, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-module": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz", + "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true + }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yoctocolors": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.1.tgz", + "integrity": "sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/doc/vuepress/package.json b/doc/vuepress/package.json new file mode 100644 index 00000000000..8644b4837ce --- /dev/null +++ b/doc/vuepress/package.json @@ -0,0 +1,23 @@ +{ + "name": "espnet_page", + "description": "Home page for ESPnet", + "version": "1.0.0", + "license": "MIT", + "type": "module", + "scripts": { + "docs:build": "vuepress-vite build src", + "docs:clean-dev": "vuepress-vite dev src --clean-cache", + "docs:dev": "vuepress-vite dev src", + "docs:update-package": "npx vp-update" + }, + "devDependencies": { + "@vuepress/bundler-vite": "2.0.0-rc.14", + "vue": "^3.4.31", + "vuepress": "2.0.0-rc.14", + "vuepress-plugin-search-pro": "^2.0.0-rc.51", + "vuepress-theme-hope": "2.0.0-rc.51" + }, + "dependencies": { + "js-yaml": "^4.1.0" + } +} diff --git a/doc/vuepress/src/.vuepress/config.ts b/doc/vuepress/src/.vuepress/config.ts new file mode 100644 index 00000000000..d1cce04bd0a --- /dev/null +++ b/doc/vuepress/src/.vuepress/config.ts @@ -0,0 +1,32 @@ +import { defineUserConfig } from "vuepress"; +import { viteBundler } from "@vuepress/bundler-vite"; + +import theme from "./theme.js"; + +export default defineUserConfig({ + base: "/espnet_draft_home_page/", + + lang: "en-US", + description: "A documentation for ESPnet", + + bundler: viteBundler({ + viteOptions: { + build: { + sourcemap: false, + rollupOptions: { + output: { + manualChunks() { + return 'vendor'; + }, + }, + }, + }, + }, + vuePluginOptions: {}, + }), + + theme, + + // Enable it with pwa + // shouldPrefetch: false, +}); diff --git a/doc/vuepress/src/.vuepress/head.ts b/doc/vuepress/src/.vuepress/head.ts new file mode 100644 index 00000000000..c9ce42e5555 --- /dev/null +++ b/doc/vuepress/src/.vuepress/head.ts @@ -0,0 +1,10 @@ +import type { HeadConfig } from 'vuepress/core' + +export const head: HeadConfig[] = [ + ['link', { rel: 'manifest', href: '/manifest.webmanifest' }], + ['meta', { name: 'application-name', content: 'Example' }], + ['meta', { name: 'apple-mobile-web-app-title', content: 'Example' }], + ['meta', { name: 'apple-mobile-web-app-status-bar-style', content: 'black' }], + ['meta', { name: 'msapplication-TileColor', content: '#3eaf7c' }], + ['meta', { name: 'theme-color', content: '#3eaf7c' }], +] diff --git a/doc/vuepress/src/.vuepress/navbar.ts b/doc/vuepress/src/.vuepress/navbar.ts new file mode 100644 index 00000000000..680e355bd6e --- /dev/null +++ b/doc/vuepress/src/.vuepress/navbar.ts @@ -0,0 +1,7 @@ +import { navbar } from "vuepress-theme-hope"; +import { load } from 'js-yaml' +import { readFileSync } from 'fs' + +const navbarContents = load(readFileSync('navbars.yml', 'utf-8')) + +export default navbar(navbarContents); diff --git a/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg b/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg new file mode 100644 index 00000000000..5ab7d175fff --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg b/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg new file mode 100644 index 00000000000..c62c8e1af80 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg b/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg new file mode 100644 index 00000000000..c27ede597a8 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/blog.svg b/doc/vuepress/src/.vuepress/public/assets/image/blog.svg new file mode 100644 index 00000000000..00fc40d7926 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/blog.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/box.svg b/doc/vuepress/src/.vuepress/public/assets/image/box.svg new file mode 100644 index 00000000000..9e6408ed046 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/box.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/environment_structure.png b/doc/vuepress/src/.vuepress/public/assets/image/environment_structure.png new file mode 100644 index 0000000000000000000000000000000000000000..3ce82bbdb55091591bdac1d08a42a783b4906682 GIT binary patch literal 235688 zcma%jWmJ{h_ca_2U565s?pCC`Qw1qSI;Fe2OBzAxZV;3Z=~hCzkrt)9n|E`s-rx0( z|A%)BaGXJSp1t?lYt1$1T)a|LkikT|hXw}+hbbp3sSF2)j06V<=|KVi!tsi8vBJT@ zbDK#>C|XF!NZ44~*sIv;8@?1XvN3q6EF=DqkB=7)jv>}iPfuBvg`rCyS5L32kC_q8 z-bFbyG(uU=r@y6_0-+-tqxe<#XT&IYcwr1txE!U{PZk&v6z3K9yWTN=nePAm!qSv5 zG2Puxv&Tf`$q|DuGwny-yRtX%-#35wXX7FveZip6tZm4J>)3{`ka}Iwx`{~Pj5Fvb zdVB{aK-wk@{~SSo9=-$>)gGxr1a8VxPLCU}>wc3XiY8n~@Hcdbe-z`HToD?@yakbl zhK4DwEk}1I&Btu0q`(TH>YbPQCh-is5DBJdtsD>3pUSj$wR9!=!Q*ek#xf9E7rPmp zktl=VP|EpmF)*$l-z9o+P0{hh{Vvg+RW`QWr&6IsciGucRyo)5t^Zsf}sa>nxVa7^Ge3LJch85|V6f(M`Xz$e)6OvvxAAR}cW{C*AT zx&3CBW;yshQ8+nCF;!Ri%{1ghm9gW`CoOD;bGcy*^10NcxuH~Hn5rx&lU|{XpH4Pc zZ`8~EQJ9#R!ZQ6IJyV2VYm`fv6ByZ_-&pOg4>Y3Vh_jD-cjc8dGvnUBGiUU=zW{DxUg zf$3SX5m^u#fxVi^yf_4n6)?P&rYulHq*<>Uvk@iJ}ZjfwXO zeJMO&$4fOMQA{&jw!XcwKqj}Qm?j<=FNK_hz0W~>za_}4Wjm5A_SfFUaKYDZ`Q9}qd{xg59XSDOFnmyO2E-5W0482x_q)t=dqr7k<4lK-7DC(>qg1j&W`TaJ?bC@&tT{B zXo~W`&w%TDVX)krw12VF{56%&!3adX{pw&=oU>Ymk(}9Jrk~u{vZG1}369RD{!2%%AALQD=T{ajR;1PHPDeLM??4*bk)k^BmBa&k>2`&_>kQIie_m8h97 zB_qyn2Cc>XpWB$i1gn zZ^baRFrwB*i{jj>Z02C`^Z;X2*?jUtmRv$`yqT>;=6#BQ-$_N(Xz5F&PAu$3Vx=#|W%x!Zm+Ygse!cV7 zZCa9)`Mtaad+<)$-u{z<|5@Jx>H^*J>E6=Iwm?*Nw|yf%&(jYsh6N*8hMiHvSrW+n zqz#U|e_wr3h&o_D&{B?UjGhY$so9()Gr|8p)TB zV^qmUQFb&jHg6eZX#GURQPO@|M$-QbqF;{aH)PED&n3BpLOuUlvPyZ9pYZU z&Xo$k^Ti2nrS+~!>f*BX2diH(7qbh!KHqq7m%}(h&v)QK^Ci{YyLW@-K#GyqZrdoU z??(x8#HG8;6Nmfz7jP&*U4z89q5LqOPsEcK~j8_mPIkscjH?>(yyv6DhSJD zBf>U`_4)%8_z}^6a9mS9+il1GIr3D5pa#vd<*w`2$B9QfbB%f8L1@n08UMt>cVD=5 zXpS)(67=fwV89<WZgpXb@WT&q7K;*s9onI?!-Q=V6y*c>2R=p*&c{*$-=4#iV?U)3yK@3TCNyX=zQ zS4V0)k7O~?9>StQigR0W_+Q9)qw49wkWeUtPTPXe8%eFe<({$s$_k;UREcyiw{80j zrMO?_+~;-#ztu=Tcq^qxzaN%X_FO=B?=<#oC{*6`%_T5c62GSbeK{co0f2 zjG{;s5q;35^UuY&eIeQldr@KZ3I{>t`qXghlTC)1%ij0Tk+8}4x+44t2Fp(>vVMgE zj0|#G=KBQVx;axjLUN04zDS|OegGA7k&o@<5u}y^u3#hLF2|2)o8~vDcywK0A3K)z zQmg2-+ChCxF!&sH*X54#PY&4@OsFj;bty-O1ofZyDMC4L^$-~| z@0QDf6mOZ%Q=R<#BK?)?4Z=j--Rp{I56?H=f3<18TDB4x4w&^k=%afEGMZH50pIUz z1RlDmsA#{BXTEDNOXB7Jk8YjvJh1>IcH0HfwUGjBnhP)WD>a9q2-;5aXMIN>TZ>&S}HwXnm|guN9Ic;8KRu z44~PD6?Yi`fgX=Q{>TGSGk2C6iV0Gx;gVqan_s*4_pOJ;K^IUu`WkrVJyp1^XH3t1 zlL=&zk;Gs{W<>w{!5C8bT!RjmFdS;>E`S7k*MIiBiJP{rmq)@Nq9{nUf8<=hV2|6y z0nZEH8A(k4zUjQUDBR2?RP{7L$Gzxqb*MemU-oksg`X=GoEqh@pwG?KiE#^3hdA-k z#zaE>cHQH$=4ye*XFaSZ+5fn(q3V#rJctX0^&Ba`W7lDhLK^=vvf+G_ z7b11hzpzn^3Ir6q{m+EDT^8>f=^U1OAErnIuFcken(;cB{gU-&#LM0bw<`(`+BzU| zqm}M`Zl>PkS*uQ3wEJ)pbKWWzF+dFRN>UHt9RFdMZ6CVj7~?J#GBz1! zh*%#pN+3~Y2Zi4ZO!VHRz)pk8mnc#T?)N&4b(a_OH{A1fadB}Dm5;3@Aa?S4pf2xE z8b-SzijPitUTl5Jb30hJo85V_NfOlhLeRpJujE; z-?0j_fF5tpGTSul1-ordRd#YVgEF#zx)h!A&8B>uyqbp}8pDmxxMw#@BJ4R?1EFM> zpy^oPK>}$<58`?gagVDf7k{?Ft@6h}hAt8oiD^a8hg3;D>sTFeeYW8aw?FYnC<26m z`vxR+bY*5r0G22bkTBR7CB7y+$jBj6ChEm0`|$3GJ{!x6T9v2Q)x zxp-b@KNDz$gtXVkO6W7Zj*~deM;T7jZ2LaofV#t7HS_R~{p}$48!J&u85+4w@_L-6 z-LGloBMBlPtj~&7qXbgjHWNvdV(k*{lnuCC*2J&RYKFONv&bQ${czQtp-6y^vL;r) z4BtHaj?T9~ZKGW1WWne%>!h4%@L6j+CgoSj^A$iL5pq|Z^SSYITMQzE`eQ?iq$pwB zR^LWRBaN7~B;XkLY-$#fraoFEs1~cr1!Ivi5$d>RS$yjZjuBwe$C0q^$e}boc87UNrHg}AwvdP0FE()35{$5U*tk{Rj9BnQwFguH<>F> zav)uKJ&c(~$|mgL@1|}Xm1L;ur!3~#@Z$&FOGgtd{?7CJ71|8i?%6CFM zF~^LN(DkBCe^x@{u%fWHIsf3U`v|AX6k)M;3d9P=P>c50bNJSy;txK7ZaEF&tW!`g zwKljNK=0qlC6>}|8()9hnMp7G9miME6QH_8{18S)MrlDZLDvuXtQwqqdLKGzV04gf z4GA&>y*N5~=*KleGwlPLrZZ_!n%-`RSU`|V<5e2IK7UL@-M+~#=fg>-R!Grp^-Xq7 z(>nHh8b9jtL-8k~Y5qmY%2pZUXS{_uP8<{OVk0C%u7#1AQ0UhBK0q*uAK^Hu#5b)o z$?0pS!7|{sqyDPm6v@!)LWygU^*2QM2=VMkJ_d+a+kSqEvM$Ixl++JV-VN=D5NbQd z-u9sYOBn>BjbbFGivkoUn+c-A%?^BVf1m_e z3VqatWfW-}gF{ZqRsgEI;u%$pKwWerL{w5ylv&*&7S@Uai$~3-@b%z)qf%El$!^^D z`bxuPH?s~^=q5B6)k9o-*3RisuDSW-RWkqK2Y!*X>`Xr7x*3Pl54o)BR3T0@J97gf z2~}Iu60>OA(Z3!bTRPMJXAiqk7tF!%ol^K4gq`sH&wJ&vvitH!Hpx4WtX%&$+F1+g@39g|fis<3_UB zkp0OOX+2&@40C=exEBa=y3BJkjqc&({)3zAotr1xt;w<*Nu6DzRk<>wjG>2Fow34> zBXl?fkf&&*wb*L<`T56>M##g@9=&<(Uy$~tDx2P~=QBnw_On=Qg;og-H8!u$Xx)-% zYCc(9k*L{_htSR!jU&>qPpdpac0LRU_A& zcG?zV4Wp|-9U^qtx`|SE+C(m-KV#=nA3*3Kkak#ZKrDDjDu1Rkq zB{)&8ZL!d!h^|{uf45)o@4NE5b*xE-leW^aZAJ7y%t9Gm5SvBLbrREVZ3xCeomHT&u~@GhQKv#=y;#~^oO@|nXwDu8$d+AbEqEv0DzA+dAN*Ew&(`FUjk9_dJ&E` zraxK`Zc8}aJo0U$c!2N++LG0SAdj*-gADhz#Sh^nK$ObO-d@yL-HBc3 z+kw%iFn<|JH0#NQQpUue@Dy0)j~-)ph*@c}ceJjL(K7m6v+yj5i$LRF7LTiZD^Y0m zO1?Lm6Q%f+q9Ai%nal}GoaD*$r>E>|B!^k;34dq%J#h+JS}A@|h8o?EUIV(Q2Y~h; z=<@ZAk}4MMLN6l9&Jsn&bj_;+0rU8{*!xrM&Gp64F*V(n^4*CdC7Yt}@I=w(M>si9 zB`;oS_>lRv7Dephg5%R^ni2RT_tdR!fdv+ANqwfQKhNrUdc%64ay1RMTx6PrU z$&f{SN%>ZvRR81z_S379%sf708WpJ`P6Ctz3ymR&MgkQhAK$4L&3SUq?&)%0#Zwz= zvQc7#Bt95oF zS-IRm%50{_uECLG@q7;~=&ll*_Q<5j`DzZk>0oA{VGCBAuO=#}_g$kEGGCH!2_9o* z5{#Xx?Eh0L+-L-<9MT}h}9(`Y%seY6(u{c{V ze|)2x$;(t!@g_8YOx^=^Lc_b%%XB!*{QRyoH#A1ru@h?vS1^~SgS}FYJb7#`!a;FB zEI2&?gw?+W;EAq2H0rvQM3t%x>1OZk1EM_|jM*vE} zN#SLh?9Fp^(tN`@Ab9X(lQW%_)(=YQ8{aQGD`?zHM2+?Kfv%25m1!(RW3(p^BZhQ3 zU-UoJ*I#{$61A_=&Ps+)(KNv3ix6t33jyG7;2 zNVC#56zyPK$U;o^Vylntt4J?s81}WYgfdl*T1^`@kK+qwDhx&tFw<~!9cK9*hgE&Z z25IJyB)8-MuSk211Vb|X;BWYDkS6*Hm-a`pMOiFIKd%Jc({xMa2=DA0wM7!ib%Edu z)ZP)tgbVAFW?ysP=-3kg1VB2-HOq?gYhjaorgGF`{2HXrK=4Z3gR+{+c+rq`Rs z4u0ep!1}9iwwKleS%uDP7_=i+fvE!9H4@U%WUS9VKG!Woh^ok8LIaC@n2X(4qB8i& z4+LhC$FXd(u%ygBVX=DL_sjr1$puFO#1qf+B9h@`f3vR%kH54AL~-p4n}NW6q?C#f z8o-Nu7IvraBB~boS8}su^i$SYsj-=FQqB~G2S{4?$(2~Hfyhq70rV@;Hq{52<{Z(Q z*>_HF7l}x?%<-kPT+*njgP{;y(W3D75bTrz_1rs+r{BXR(iiC#TJem;USFFY(#Yyg z$^V%?c_FeHfyqq7+?KFWGoptEfsQ+KLM$ZgH#mqc=I4r`{^^S0{{7GC=^tM|dP9?^ zExF07qyAd_1^JL$VPf)lhD(v=ryd=!M4!FS^IV7riNc(EH)BOfxjAleTxlA1 zjbD0-;;8sK7|jsKQqX^c*D$E@K)Mx4pv6Sln_Jq*B*V+t99z0vPOvHf4vAF^BNdQX zMM~T+d+-^1;GbZLr_#Wm;Q3l27VLDzQs?DkiC->7i{wVUjnfMZh9eEgTI_24gP!r- zK=C|X43WkUG=v3B+o|LN?pWW9emI0osbFFIWd3HFu~0l$qr1(XrBa0#nJY*wKR@`< z7|!CEQ?9q4`53Myi7H*^&H3YF^YcN3#XE>wN%Hap(fd!;PV|DuV^|80-h!fToUott zD_*|876qJVgoiC?UgqioKtYRl=GS|L;{`dH^LUSA!K_xsqEeucmk7I3X}c368ih); zM$r*|zpe}AGMwA|Ex^4728zi6HaX zd?=?@zC5mJ^X36pp3&NHUiBf6nuPOGWh43GIutvsY?6N-T$lOx`HCaB=zLaL8Zooex|Dr|wOmPy~^!@EB zHpl$fP12m+W*+Nt^%=DQ@W>nXt=fvYldJKHt<=S<9g|108eZ@oXu)5g^rrHu^z*Hd z$c@b>0ZzMItDyOe$c-9MxHgwHzqOLT_yPaO9zUk30T502ymKjk&%$A-@GdStz8C2gT$o(D$H>ER!|)sc zA@aKJW^!76QyqV&P7P>)7(l3Tup=)kfiR;IU+`j8w>#+LdHj%7lkI8c*eKxis5&Dj z0@x@m8q(E|!~c~VDY8&{C(gY~H9oRu>s_56A?7f(KYI$(6XM6cfPpAv0PuMTA5q?e z^QHOu@z!g8i?=vkSX6mt@sd5E+OF&PPl6Z`S^HpOsAF&p8LViwE6s#3Qi*4lYj0T(iQnzM#h zKEaWxE@j|0DOb-Z{u9k6Eq2G;24YB&kx=q5x5LE!C8j>z>_D23J$FZmj47mFkq_qw zTIH`?6I#?jjg8}TSnguD5{Zn@mQUuS%w)qom2Hj_zB(NFBFAa>U6N6^$&)Dhi#E8A zlHo128tew3C#wKVJvGdEW1Lk({?*x%GEu~tN*C6@lOPSd-}|wou}YJte9oJRcg4pH{m8F8P}JU;52`~nouGc{eg*iy~B4`KdE`a%X+&SgbhiyDtd?uir$xIFSM^-$X# zRiS0nC82#RCNK80NpmD*`(l$rPPbN>{bXkVK6ke)IdHZl$MIl$wqm^nzMDlyOTlc2 zssZ`u+%F=rg%-NMGVmZl*Sj`%<0J>@&+34fW4=|~0driJ8?-kT@vP)IXxX(4W+VVO zdZUo6kPg^rSO!#CSvfcIZ4{BPXU!$(fQB2~9E!B-ojskodm90qG;Wy@`U5KmhQmpU zMm*|_EV$UNe>&%VVb$7KXe-lQ=M;%&vlT^JA-8e&?f4Smcz|?4U1r8yF)srXmT@6( zV*Yp43VvjC)lw!o9KEFiuB^1Y?1jC)l$Y<*FA&bV22=FO2jdC6F* z+xph0#}bXw==dzF!E_avzgQsHLkRF5yahE4ja}eLBAZy)hUndyLF2$E(R}ZfjiH>7 z%qHq4ymqcGK$eBtfHbL5s7UE2;0lO#dyr#@J~?}y@ECMK2sZ#$1%F_9SZE+=7Q|}g2OWAO2Jtmo}=(lw)4Yppa z@kxPL%fGog(QKIS;}6PtkM^%l=2P4Q5RU}#&9T|VB^H3>6N@c;Dr!DjNC~*$G~Wk6 zN2Rz%)r6^jYBg1N`EK*0Nka|yAB&PqYYD}n&h@#u_E#DPI6LBUMdap{ zYszb&t%dUM^^6r!Ppf)PXLBKcElMf%t?rOFlZvCjDt1lEk2aELPR{0u8az%!f0o^&5FHW_-R|O}S%F=V$Hzu%C7GGEg z1^i>Z(L$x+f~VxvMvdp|CH9~*pO*6Mr-b*s39RAwI(OuFfmJ4UOJje>Ap*?YjN3{Y z2q^+}z!t`jvOJ6cB9ARu+5&&q$3Jn84$6}>{%2*+2=7R?g5H;DJ)5G&pvu^-l-4CG zOl;)vXy&QNn`JXIWyyk4-oiK~%;bcB$5GXV2U zJf%c%7l;Rv-uyBh6Ck4N=-Z}*eT!&;DiYplfH{r&ZAH9$%Qq1|eh6P`V?Lb@LU_Zt zW8{MF8FJ?fI@?zdWn1JjU3TW|FHh~Ljb=S|8kKcMDwaG*saK$L?i;MsXjJmd!>`!E z(sAQjeBlwG0r$Nxws>0gIC)X-w%$_bB=lucf9T;#v}``tp0$U=kNYt5{1A|PfIfY9 z?3foj{i%(mTCt`YzsTi>Cu4{{g(X{-1WRG-j>!(685BIK?iePc;iv0*$398p{;pS` zP;6QIf+Uck&}yzk#QDS`?A;*&v(^%RM^V)e3bZn1r4M@&*m4^iWsg@6vj|3$)+5M- z+wwT~@gnM=u+aq&2>g_QU7On`w{RXD=+k=zxGk~;;bJtU; z+EV=i(+a0Czpg+*BeA4gHeea%?(h~gas;G?2A%X(u7`c1x`jyhZ9ku*uekY74p6}B zLT)AH*?N~0;z?QbXp4NHS2dOFIo*K*zF^&np&3YC;KqEub;jVv!6|br)e~!-yKWb# zKL#$IghV#`Nef_&f_@qMN8P#&FN$F(1(_CL#vm?VxYob8gKm*}HqbIUg#2Ye;q=cL z`t%q^CbV89yAMhH$g$5yxxQys(|a8lcMn#nz>uYaK7A(@KvRyayI*Bhq*>iX2AZHX zYVD0x*D8f<&9*Rq;v=p1Lx)*?WNwlg8+{6)9j{1iT`9dx>(G{LA421#VQW<6s-0W+ zYQ&D}0(Y^+9=^44%$|qw;{csBJn69kXayjIXb_%6s@URMy;gr6;eNAT$0lb551!Ed zz)Ja`eZ{AocjkD=y$)kD)9+@3W(qz?j2>^rG5#q5_6|G57JYlBRt505G`B2OcO3m= zU=h(Ce{{l)e8cguFdsM|j0aOaG<)6xkwns8RGT#O9Eea~@y-PfqXf3uxRCc|{bRY{ zFFfvVe5{H87y7&1JtnB_5&{*xb%mki8A#y~)2vL9-OHJDrG@;R{36GR@uQaGG{KwK^>FIoo*V zS>K01IZQOGQ;A^Ju5*<9w$-4MI>K9X@X>*v3KRW1ML#!RLBQl;?lb#3uiSZ_RvqGg z?~ccPgK-LXOacR2&>u8V@XR{K0m<#f{n{V6OH|n1QA?imr#^i4KV;+TQWzzdE{!TZ zXkri`M7>URSkG_epoW8f0lrCgJGDHvM4+V#{q0rzTdcI@gO5g&^g2Jpsd>>4D5Nv- z5xKf6=^Py7BN=rWrx=3Tb8~gYnhzvu{m#8~(7ZwsKu(4!ga*`qte*DE@Oj&7*g0dz zWg6DKq+w(EDSZPWN9J)bz-sIGk^GfD{lwm)*{su!_}dvwXz6((ZJKLb$pevPk6luS zGBy0-0}6@F4UadcoL>*%&@ABlq>kWNia#M``o?HTeb!{BgWbQD$ggktZUbdu*@<4> zBC1svRip2KEF1D{1iIYYuh2j-HK}@FFkv7#qn0{cGIw zKVgwAy#UCgw?Jk{57r8Djf8XB z&!elAP5OdLdbp6agmaObsu6`a;1Q9quHUNO}JxXmUoN~eqm4kFu z1nd)E9B*+1ARAQR;aGA8wW72wXZ=|MUqA|n5Rq}#sUwE(IhYg<-g)eo9Pp-UPJ)oM zJn1Rv%Su&2{#;Uqg=_^;k|$~|b9SX$l_p)VDFM4|)7VdiRkv*AMZ`Qwi}_@(64l&*wN10V<3iaT5$tpJOOar}w8N!rWx^=G9TV^x?L9-t+y#CiC1 z@>K>Q(CJNl(R{2p$?UKam{syw$%_Rh)k0gjX&=SM@N1BN1O>)E2+QchUU?Jt z5b|bDb)8}P9I;JOyR@vliNr4a_2JN~U^R&v~Ba%l+_^Z|!F(0xs5y z@(K$HOC30>rvwzY|X7f~+spWj>y|U5_xZCN3V7Tsx?bLwnw`GS7P+ zMAnkU+@JIAH)(wKoprWk{#Cl?zTAjijsL##1n1src_&Q_FI2~4i%V{3KJC=!RjUSo zBr}|*JhsfWfq&#|+|ZUUk;FgUOQ(WUCLc7NW~aK$xjmfnhoD5 zYtHtcAtB8lwhljG9C)jAK9n2;o*rb4&k-6vJB=ql3NpN3n~mfL2_BJ3X5N4vL^?GP z?kDl;5`el!r-BE)9MTk6@es~LFHBs4cyv$L zs9c>pzvz*8G}#G>DFz;l>H<^vV&IheSPnosd9}iOWCxKsGST-aZL0Dkk<>2dvGHA> zip3uheA_yAB^T8L&aX~#o(WC)tA^u9#pWB&lYxm^YF|H87~(_T#iomGJgrWmhSpPZ zc#M{omAg%NOFl?{H6NEaEiS9|J5AFV?jEaIA-y*eJ{xxF^%ZBkp8qBVVno~US|YV; z1Y)W&qbO!5!hUuYPzBjxnfJ9ae#2qL`d-~&(IMvaBCnWjMX5k8UxB(-laT=V87+o@ z`%JIAS^O36bUHyQ< zpj{t18z;U@+|>^{5Z^? zh!P7PCeA?ONn`}Ck0Q>L8qZ@>S-{9fV{V^r$6p;euMl+;D z5BW}J9Qq<1P%vd5o&lhWCi&p-M#hrDt(a_P>AM+3qV?nUv%vS|=Q3;_+ zH6Oy72i+3KTnXs38V(djs(`U;#&)1M4Yu0S^#y?ahya4Nd#SGrHKCmp{d|xK!qg&I znq4{qUxMwG)8?d7^_q#`cUAe;8Q>qL##$1(1tkvJek+(fGkPRG*MV**@w^$X%@nqR zo=<&kp`BD^@+e8J%`IN&4@p-rEj=qS1JAEa?sbSzQVy1=J>b|yh!Sgfzvtb)f-4ar zi+!JH)ED02FZQr2tcCi#9xtGk&YYXbD5S-OI+z@CE)oAPi{LHWv49In9>%VVm$6ax zsQU8uvBsMd?=%wjXoVnhFeoScG>hC ztgJhv1LoL0KY%Qm-h6$bJ1|^1aLmG_)F#Ao*uXN{Y2f9V5dXt1=Q-=Zp?F9xsB5GBw{pqGW8~Qnq%lCph2ES$zynbP;~Ln8gTMlY{yCs5HgBPK9I> z0@{XY^JKATG)sDxPV`clm9@FXdKaNaKCfYZzO*c!kBt{9t=|Lt##&dkjV}UBoW$Kd zr<*-atfRxo=Z-E;c6{2JcEy9RckJckT!-y@2M=G95%l%CuNNK`!q!|GKZg@~oaAk@ zHg8NOt93;zCb9Tb^lglBYUb(WB*NI-^z#N@opxY{Wl&jV|0$ZfceFG+uzFzh5U%n%5l z>s!%vM-`S~{K}0R**4eN0L9Na*(x9e9e8SRyFNc@lbiU2wok7$pUge6V?8FdEL4q! zw7bB6M5r+`oWrQlxN*MmxNzR*`dh18Eqx}{M;YYcM2FM&K3B{q0%4V`dsERTvH84F zmFeXI;G7q!^n6q{;P{*8P1>AhUVO7T4QW}Y=lL(;;Q_JTY*7L|L|{XmbA^b(kMS0U z9kliPYnPLT%E~fr4^%r5iVFz_L*$>`tsRH*zOLhDA4_w5qAevLaw>Xk@vXUcbQF$x z#jctk@fm)+wg$F&2EPV7=d4zKPMH?jN%~6P7WU`Jv;A&&`R0ecDG1eE!wT;PpIdV;Kpc=r9GY*g>rWr!p%7+80zjHP`oC88NA;wmhw#Kj9kp*sfiPa2T5M?v;_R2SgBJ99>M^$X!|Z zI$#>+D3Ol5XAe|40YZ-gAbQ%Y@3gY!85Eu7`yZAO{eboplW0^oN>oV~z2_L-M!PSY zg2n$#J8qNdtV@!+W|5@GJ5M9a_R4WdAmD?lrqSjRo<83IKb zef_lc>P~6bwpV~^Ihd5F|KxV9yyH%|-hq7XW`khes)(#rTlmWOJz>31%@bXpE2oQL zn^G}e(imw%_z>oGk?&WV4-}d4zLx7k%akm4A{3i$QZIfwnxeE|hE;Xh7BzvMnce&H zba0b5_{5)cq!1_Ef-pn=uLOV+45z%){tWwsYaN*B>2{lKHs~KLk^CXrQ23$Xh%!AK zC2Wga09p}$zk zMp+I+VmmqL*-^o%2H}*HaI^IM6q|FDXm2K3(ORgX$wMuDW7Q{zpbi-2xF7- zkJfJjrzU&0zf14{^f-jLO!jH0Zzu{p11|hSbjJG#h>}7xEHdj-xs{Gc!+hc_VqsEF z!pSn3@C4km`3@H|3QK|u(s;_z_jQ}J?s1yN{47gl$A=$6wi2ZvEvf6PnAy^H znSH5TOw~uzL6wXD`6yUXtvK@zEyM}I>Yf5e`v7?^Pv9g$6u)kk`~1Sqs?W`Z?#R;_ znCgn}d~&Gl$t%t*6s^aN9w)~mOEUxF?$2jZG4P4-;nInSd>t#B_WA{OTut;BU>*DK zAUiJ`GA!@eqim?>DSrJ5V4}0F`Zj9eh6-?jyE$M1JE^n^Id|f~d!r)u+L(WZ{W7B^3eMD*NU5Ye|`>0YJ{7PnBoO4OahVM8k9s!C3 zZe2zdsesWZG=Jo(x9YcIJ|!-i_>gc7&)#HyTPMF0G%b(ozGkrptGdOfhxXENZRlDa z*JQWcanYKIw*REVt(lz*o=e`e1tIG?F~1a}W~Nqj1T8tiwmYj#-5K7l8))Lxk4d#3k2;rfNSuVxDeEo&%;pM=JXvaCg+>jiuWUqN4HVw{CVDS4-cyj2)C zi%~qXePq6z;-2`WcHWMi{Ht}FH-deYD}(}Kef67=cZx1p@Xrfx~^<=LsW)!M>mqeo=fg2!&p6mf4Un4jx6m z{QA~x*a}mA$2<_{fGR}*nTi8$#(OYRVl?MMl;hEI z6keB41ogjH_ID*lg&$3m_A(vSk`BiD@(Mb0cyZD3c8}l>2Au}QuarrAYSh|RKXCs` zE-}t`V>8;T*m0$iFskmU-T~pb{PlR2vJqRXI?Af1OxIC4`zL8M9!L4?$`fzaftY`| zY-uflt|j~Jc&kElB#?AerZ``;cNd4>zJ5p1(nKb*Y~tXcAm7XCav@;&xlKI%MQ#Pg z=W^Va&>8F^*z{Ai=kxu*ObS4qPKiZMh%ni!T= zC7t4RclRzMMgR^)nt*SYnMxx6%7uc-QBtUCgem#?D`yFewpH8GB8J=13#&Eg9M8?q^@(jq7kp%#{fG^@TYC7rQ!m<%h%8gxmXqdG z%UZ4RdfmZ=qZnf%sX4t2KG@cGDiN54Bu4+_28}qkz}?p-C}Y50-0iG8Ub|LLHQdy; z_{!H6dq5P5<7uB=?Irkg_tNW;S zib0hm$f^D?#Ai^UTm>;;a7(F*_E~o`dpBXk%!|l~wtT8OjR}TW@hjZzoil9Zp?EN8 zOxb9#6Y12%lu$cq=vKSe0S&W@k+W6R*Tsr-ey3u=!$Apl>lt2-i^6<{boQ5FZhA<8 z31A|WsBhKhfQ7RUfFo1jk67-dDUILd^(SB4glh=9|P_w=m~nG@ii+M!{lF`W*w}B%JU-E7f!znBl9Y|eqoW% zGcikpWn9b)5!(r^%VMvxs-Z=;Qw(oG>_ab$$C<BM>=Ri(GTO(qEUpZF$C+(72!Z zJ$e2x2qy(qyO?|&aVltOVw*WGm${LP`T%-0eJWNJHQ0jy%0Lr zcxJVJUIJauJt;RJp*U1rSWY;l8;+KL?RmLkV`5s;Y`Z-%Z@0{!|7z)Uis0Gd3D-s*nZ|+eQ-)b480vE5%U_={kXjc z^hCrWZ#>#`uVZxw(s}odQrtpk%j^$&B7iBke+oB0py)y9PoWhsVK(d7Ygkhfk38_V z4ePhPl!nBwlQWGDVojKP{iv!1yD}`ft}OgcP)2dvs30)bAd-73`CQOsKZSB3k8nQI z3yvC}hbMenT^TsO{^${wK`L#kKNH2Ka>0U*myzjt(-WtVI@-+vgC%^`knX8u^Y?GA zZTrce%+Cy?Rxi;&xHz{mF?m~ndGzH4d*^_{9!)2)j4MKJbq7L52~3#+-(VMUy=%&` zP$C#3_71B?;;5&a8QIM2b9|`bC^v@cO*v5c-T30?tD?)%+ zT8`rPv<8oSB--ST)xlO!%3aX@py$_{%=J}JquP@3mZ4L4T7^utotL0jau_2B@T1?Mv^NRf)eI`^dsp{G4O{OChc02$%0@!S#s>ra1mgfr$8sHVsey9*r%1)uuyK^V+rhTv`F#maJa}4js-W%MFdW&6Vqho`sy0mzE&49s|{= znpjc@POVi6K`1^u>a5Jp7Qq)_lNPINnf}r>Q!W#N5Y{)T(9%6AE z3&r(=yT;D$XKV9rq7iSd+@KC>-9)kD`y=B9zG`W{xw5HT0Tb5qxohYn`UVD{o|A08 z6Zsal!O$-igpO`Kg|yfm{^EuL%l+0M{C>7Q{DK8>eDJA$oCdB}%%GzKfAJpn4Y$ux5Av?UnFzLhyE zjJjW2PPnwA72=!#*6}WnytCDp6-)gqEwL9$4?cPMq7NaNx3`VqF(^`FnZZer9$y+X zw;vraB)S5ADi_!uAo}=DL{_{O8(xP)GIT=kwC}9CC1S;#K294ghpA~hy-_Vv{>(|> zghV--eT6bQB&kDvO{IQ{u+`~L?^R2l%Y!+6CXvfs2?po82NytrJSd1E{dcwvm4X0v zJLlG)k#F4tcs;{%bV?hJj@JEklFdS@XRnycVx-*L?^L7^e{MB&-##{#@3?}YkbF#q zL;RSg4CbSRa%8?{fw|;q#u^`=F1Me~A*VQ|l43s_pN~v%*Q}74IrVY9PJR7)TH9WL z5Vr;l&o-2{Q9w7M_WKW$ZV73e-d#^%oj7H2T6vs1*&b;asxFK9<{1f?!1It0N_2m= z#t)-V+?=hSs4sLsTAwqUM-AnZ4Mdi4(wjw9NpDIP_VK@N+yBb4|RJIG)~kdW3?>O zD49_VB!jm8UzFWfW)^}<2zMT$oU3~^EgR3z1Ou(j4mEdmhvZ`R6!1%OTCk{e6(Yx$ z1)4viTrv#K+LYQ>g?O!v$Oq5YBjNU?XBBXn1ILNmR`RgDI)el;Mo;$-ej<~;cK5F>~(26{uw*QZh2AIUe1iPadN=i!<=e-yg3ilJh@*$!)uYK}A(__JvbE2SaT5M8m#% z%q2xnXL@VPfo=MSQJSVr<1f6Wd)*vFTr1wrg&(!%BBfc2q9tQziJe>~+iHm*<5Z z3w{WpuXrKI1kEpUXwTt7ma?FtdgQ##V#d@!;kDZOzUlQSmj#W4jeRXG1=@G+8{@{- zQxgD0Uz}AB6L&r#zgNc*NPKzx#myP3Q7_v3GsC-#IIr(sfyD0!wQT)1!B~WGe<~xO zzu(-B@d9-@@-Z4$$wGAshTTPmZ+jQxy{R)kH|OdRzDO@#U4e_YW7lcZ6hF;ksZKdz zOO}E4MqotOxH;`e$Q3xiP&(r8ahWH&y$DuK0dvg-Q%Gx)TcLIw?R&^;bg|Xg1_R*F zV?=a8#4x9$zzF1S0~oQ-144lRH-Ak>Sj+7Q2Hu zcn|LTw|M+NuHHJT%5HoArc)ZEkyJ{$J2o8>igb5(cOwl-ND4?uNp}g-At2q|jUXYs zYx_Ot{GR6>I)?n=aJct)X z1G@V3%r7hfnGU8|ofbZ=h)BF*v%MBX4+eitft2qNPW524*x4-6E4mETt0Jl~?B@x? z#CfwY>iAG?&g5?naxK;`YbK7P+*ns^l2=q4azKh=0R>O=c# zgkT{FhKWw6k<%ohMDHus6+WUN`uuusAN~%<9uG07H!|3pvo%WK&BE@;(Pl6rjV$v6 z)|TQ0@=o_!cf`dZinO&D>8O<;Yw3STS>NKdCu+u=Q?|U>pPv)1^k5@@VVB*{a}(j=hz{E~|27$^hS1gg>F)73gv#$l^#q^k%sKIt50t>1DT z7W^6p_TBiFc^~4C&z_bzmwRFnQr(g(u0C?#KEc5&G3^>Bj{Zr${MAU(Nq@47<7}z- zEm3ITP0d42*Bz0)R4uGA6~zPGH<9_bC2F6Op9)rr?l`^D9%l#dkFCi{K-?QXVpurO z4jPPOn%lNr;Jb|yG{QbnIQUwq!v;r)k0lO=Di2fg{Y(!gR4#(?ES*fEJ1R)cb(#?JG>h#93=lm-Eei$00zmR?l5FmW~LO-iv6C>_x z7McQ+DO!I_dKGisa-#IGNH%_41D-7Za;wuuFGz84X(+p5VR43t9diMLWHAf}k-3q0Ar?ajqNf122gm860MVs1*=C1e;usjRxM0R-X%Z00r9}3|1tNMIf?yRb2&H>hfe| z#p6R@<$?7(n?|v`gKexvJld!9(NhIzsj z6}s6Q`$YzNRvLWzON% zTw|Ym)45N)OD{?@tUBI>#BvE!d=WO=?dBFWf6XU5LHaGv?pLwrUP|C)#N{W_U*Q+t z1Rp*M0o|l(AKqj{^VSooc5+$NC~q%M1JHA2Wzidw%|c7XgUCf{HKC&LILe6o2K^?I z{aSUWJasJ`7j>DW73JXm$5FWZ^Sj${OTqbIQbX)aXVPN2$jf9ym_+Ur_zHO)^-~}@ z#f~e8Xsc@+|U)BTW7=B}6OpnY1x>XWiYvi=2L7_F#dUuohiO7;+P* zNh+>@)~#2mV-^QArLS?s>QM`r!*#Gjm=jQ`a6-NrQfIqe{nwfCSL$Vrpz^|U|7S^I zp}44<9eXeh435d&{uGCndK3A>y^GZsW#-?Ui)t=xjrtJxm$I;Ox3hVpom2a*wyrCF zz{3<4IPmw9MxcS2d9hf7@>3oX3qeBdAqfIgW4R*bte9N~8=YS5XTA$9+8rxXCF*@Y z6X`h0d@WNToNwtOUowEL2NrPK1HjrC_uz=6T8&v z@Nb)YY0cnqILrPjqBD@DfU3bwJ8^n$&YtgrT?HOiB*>b0rvzt_uK*$8!3U%#+uG9B zzacS;SfmON?Zba_3pRsbtnVnd$kpPg$Vq8D;T%`r&s?fy0e$bgz4!(xMtAbbkwjOF z3ULu-Q^Ns!b^8NusU9_wxG$Kj&1nsB&t5sV_2YH^iu}s`k!B@$_gUgM!^<&Po&VLn z;D-cC1eJJH(a|c|nz4puYYS1hh%jX!K$tT)gn4hP1YL=^G z?@6XwDRni+3sE1{rD({<6u~}OG-08FW?Oiu!#ScWJ}&Mc0dWRy0k5})eg>>)${-gj zKsuhQDYD|XHvR6M+q(-&QzhQR*jvpL_11`8>aXqEjGX&e#qBO{A(tj7j%` z)WtTR1}>`!$VXlWii_EFd%-$y>=u{SSt8UCCJ^YUw99$ZBolhrDVM+VzmZhK5dP$bm@ z&mLlk;>G(A_0BiYf3LF+fs$i5TUcjea?*?eV0Fpsj9Ey61kA!y=Q`7MujsM?wQk5!D zfDHf+Q$R(TjxV5U1tBBHFR6vzwma6^<7&Xlb#W23{~y{pc4L7!WziN1sGb)V)zs9qKV+X#=H&{2V-0~MfB`D~DH(7F-WKG=ICNP` z4HU6O`<3=KD;Ddweb!~#b@GL;a#+5 zlVQF4>FA5cz-l3%FCA+N1}hxyhqIkby?aTIa`*lMe4x`F`dZ;n?ke-Y#(Sr@e8A`5 zHrb^-OUf%-)Qy4xo2+qpIr+KtT}bk4dfn4Gi;2@&c7{(auV#>Yx*#YqR}ceLbw|l^ z?FooOWH&?F3VWU)4&T>!=$8fqN5J?x0lWl3Y_gOUsA8_fqW-O`cV=aUEcd(l@x%Ph z`(H|9;5jN@X~MZ-D%)SCpiFkP_lu1V@hbp(?Z-f z@Mlr6RoupJ7bK)A^a@-P1eFla)y|;cF`7mZJcep~q{^wp2H+Tyqop`x`p1ETK{Zu? z5mj>$u*SCM6o%$Gweq9!&3+t)vPrAkNDn9MJW2|k zU(b7y+*Ej4ul225mfY|WKcN1148Hy$@Q(;TVDF_J=JjnPCep}!tT(6(NhdlJ?|c^w zf+aWkU^zXX!?KW<*n#OI@NoDmDFE|S64a>zUZD>!9Az$fyMch!v^R#B18BOTOsTnN z9LZcjL)Jods>$i zi;eN@$K=m!mhosGL|b?R^-{wUS0bE-P&KeyjFQzbOr~w67_J_jZnOmpAPeqc zgqD1ium_i}EW95}aYj##3wAK2TBQA3C)YnEMT1%%D=+OXdm$VL7KrVdjsF#xtp|z$ zv0hA#74RWCE7s8AGrp!T^hPYAS=1`pzb4Fh%3UbX*_#c4`Nfh*yalu`Ttkd+D{M1^ z>)ckgzXrYj0d;agJ1cN!d@4|TT}9(MnipVkZGj#OO09Ov1tE(r{aXXSI%b3R`+{F= zDEMZSr(!xHGeHqrh_~$kD^-KGKgekuA=c!)wF%|ufsAwj$hsIk4o*rSQMsXAuP!6F zmxuoj*8bPQk~M|H&aJ*pB#9gW6)XacrK;rTltORQ&}^8Ahu1(qKRT5GFIu9)qcHi7 zM^5R(x_9x z>HrEdozE#R7xQ@`^q#ED__4s(htNX@EyLo+j-SI=(1(iqvZxZdUN;#Z{CnbCh*G9m z^{tkW*U|HW#$Z8Y_k}Ds|vLasHvyC{Zova!vi*9#ad~BA9rvM-nhGb=T;Kp^H*0N~o7^kwN zsvcwe*9Qi)30XsE1L6c5iFl1J;Z;^h(a#c^9pT&J_i9zP{?658(~?Ml58JAQKCpa* z#Ycl?ZTG#9F}VJ1k)gcN>o-uy%K!bQMT5O-!>uUPdulCv!g*MLW^g#O9`7o5mJ4amu zRqP=(F78KQ=wPU9Jx9|wCmy)<>6Q+HLp-F-F$sm~S%ltKwbRY%CIx$5)`frc(u=Yj zP5u?eDT6u@puDuFd2who_Xvt3;!6BKf1UMxem}jalPh{ig;^N-_`^sLPB-}XS-1tj zpFNvDOq40m>+nx9xa<3W$AlT~%IEW+Y+|OiXWZ8Btr(YrtDJk`8t~w9$zin0wYkMf zlo6~;v543xPs9CFR=ve;5JBG=$WlaY?j5H0KC*ZBKK!{mGF2Hn>3D4SXp3+S)B}iu z*2+Ihi@@Uowe|%>^RdRE%#P&)Wh!-?_XoGP^iRx3A{73WQc#W|s2E@eJsSpIRxz|Z zH1EmIcOmHL1h|Vlk>A-{vEk3el)5mqcbp}U<|ke|^wgRuiPL@i&*up0p*8GyJAaa1 zCAJ1S)#sq9KtpoUbNWtD^~1ED9R&`Gd4p=TGxHX{8g)a%k+|)jjyI=&$dxq{5kRU5Xl*MlJXhyKz0B+v;I zLpExP0Ntf;dnZ}vaQs35K_%Y-)IZz4`sFJP00+ElhXVC~9QlzSLd?wzp;pCJS-@#5 zea~vJTyP}>dJG8mT|kM1+)|1+E$OX}A{q$}rk8qZ_Sd8wLW^v#!knf|f>#KqEmgoT zSEvBkU1maVFAWXopCCcH5&wPPkiY@g8Mh!K@MT!K_qQP^!HAv@9$Mr>ksLZ$N$h!#e#9^)eB?xn!xLb zPIQRj(Tr%aW^1-0jp7O*TkPMm^zOj)HwBQr(o(V4DeGs&liah)vPFNd!mD3Yg8#=~ zCS8GCGBg{~vctXP_(>DG8_5TV%E8NVb!;Kh1d~kKM%ZrjcvSRHo`2IafA-H;Fi#%F zDj6GyjAE<>aeuAeym?YqU=`JDInouZ-CNqYz~Rth*^%LPnQ zj6VQj@@7Fu=AulDXjmFR9lOeU-HbC~5RL{vFbzTV8Ff!LmImr?@R4eydIy-)hMZJDxMJ6m)Y;_@RFwusU`r%b(Q70u2A&rOJ= zD7QZ2A0kn}K;XhYyjE<$6DJu5lpfXXC?y1(d~La{?$<6*`)0 z83bA?I&6Y9%GQlIK?KZi+anpIyi`3Ii;!3n?$!*u#l~^LtfkNz79;lqXRkxt&Xe!p z*8Yg8>EUvC_53bHooF~4XXbxV3;IKl5RR;Z0_TDW!b=)&g!8?TsKI?QA%iRri_0UX zmOANYNvLzKS>rZy12WPXar^O;uL%&xyw*{zijF?3pPDLXO8&=h7bV~6Us=1txis3f zY-vHF+K^N7qqY(t9nJw4(aYm%yPc_PkqJMtoGX1$;&i z4VB7h%Qhz_d5491B}{Sw{GaX@J7ZYZ!WH;;2)GPc^kH4I9r|-u&ohfV6JkJ*w;vOhAj0 z^8dYtd;ttyqPXwjT9kmL8Vdp7$q)VrK@Z6KCk6I}h|5dYN%uS9$r17(^;3{7;TF;K zjU1UR0AH{8$WJGL4)gYK2g5Zel(e-n3s3a%zQ!xcW^>-A(v3yH${U1~7bEEI;pK#p zuk##f%n6H=`2Yv#P-EkqkLm}w#Y>7i+7c?6fBe)YFOz<_KHS_y6Z?sWy2Vxvtca^Qx*K-xjYmv))6%TDH9%(_>}vfb_{dtM2@!6j8u z{>sw)_dOKDDdsqA_k@V8vtO@_Ex;Z(23}vzH(M&w+a{=Vo zotmL`V6XdM>~9Q)nV6;oG~B~?SG1-yLKJdK?RxbA4^4-!jZV|zG?Z6bW~S(cJ_hsH zU6}Ld(iArYTqYi>TOjo)NgrZ&T3!O86qM?qn15$2KMOvfu)9EK3iw7Lv7MlFGy`OZ z;7f04@#wS%K-wIpM?swtfBt~ifCHcePIg|qBNvx4G_BnOdC+?;g1U}fvf(&w&y)qO zNGWc-;Drq^KH*fV{+S1-oHJ9zkZv@FW?o^fvi;iRFyJs{%)QW6z57~)A>LGhVD z^0tW@oq1P}o_Ucu>CT^H5*_#_vpPqoYU*}XN5P2AQ$zRPbUJ6LT%g@BZxQyn+O+nu ze=h^)y|I>tW-B=5)nI^Fe@Ul6tFEV8g^i8OIkn~aT4vrM;9764Xe`|ov-evGryV4g ztN>docX||r8i$GEbmIFBy06~9tV;kQ^g4*t4Y+1N>9t|z1}0INwZ>rN+d5RFl;J*c zZaE5_RJ7T3yV}E_F!_x1sJC$}Vg~?I)MtP$c>aMnJ2CvEdWeM+Q#jHYL@Hc9O>q0X zH}M5%()Zw|lvWlz;$i?6Y{LLhoF@av;43H?6XXcVz^0^H-9kLy=wzbK^n`)g3GW_I zn{fR52F~{SFOYm5iC0ZKEVjKOj24613J)=^*(lz%U~5;7mDa8Vy)SoBS3cj@QI+z1`wl=Lh71~I zY5Zi6E)Zrj#%5pCR`5RX>bXt5f%1DRX_`Tytp)(6HSI1d^=+d(ppW7#()To|La&8n zhY@LqPil8|U+StH7$1dcD*h4o!Rhm&Rsl}mnm>SLeo_0(J`)`d8?nLugB}?5i@50) zSlbmmQ8K|-SR3e1xFi?coIRYYcG;Ajs~qi8%<6OG4FX|Cog(1gX|f;W9O)Um0|Nuc z_O9=>-?n->(Kw6kc7|kDi{N0JFf;Ly_`){-;(TY*e;wWJz(?dANl9a zf5N8l1*});%x(*xXmK5GQ=MuX`}16D__ zyIym0NXSPWD3u%vlJ(8v8T|$73|%OgcBHJs6=nz?OK4Shcn<}87Wj+#-+@ddNOF50 z8~rIW+sWH|*3gA$oCPQdbo;2&eWVx1_VQq6>8wTcV2-|A`3`m))oJUoC6v?by zEhr*0*w`3rW%Z-F0e3E=H75!+?2iC#mTB{3xBGDbWC9}3@30l~Z%S#2g~`;I~1ZXzC?8nxm`CxBgUOid++-6VXquvN(k*@@V#opPd+8y->KC9JBi4oQSC!i%xH>uRF8(drs0MoFxx}N*;F|SW_FjRy_9Tws4C!ZEF+|a== zE$jtfn}i3#F+&otpvxH1%rf379fFa zbjN=Whu{%QtO4e`Q5>rHNjRxJTaHIP2DkN3zC9Iy3b?Gt35ma#0tJ8>kb`gl`0a+` zy_Wj*g7+y=WAK%8nz$}hnN1xqHy?PAj zH31h$U(KM%zb=q>FlsI!GnGEB)l_x7v&b3HCqPoK%a)Ul0f;pSs+E-b8{N?#8Q+K* z@xL9YrmP!M>Z)eN-}2{J?rS4Re-xZFk*YId5$3b6=%PdMz_o8g#HCuC^+kEXBTO>E zBrQSYdhVpGBQokWYSkk2acuMo+RT|zaystg&pIHNc9Q(L)3R5X(fTq_1^H7YNdA|- zMo}Zh9j7%3yJg*yHn#cqv2R@>B_NRIH%w^M!VJ0svdKH7cfTHKC$5chP*B1^np-Ud zkb&)`A7}U|QZ{-;?BAP+L2~B}{XIIEdSx0CH8d|>kt~UyYfOf-wuD#(Yd9F;KD}c zs8vQLT$+nO(OuNlEm05LOcut^`;2m{eE6tf%}U^Z?d^i9J?2MD?|+}cBJvl|)sTgI2=C}j7*-WD%*=Y6n8z81nfDgr{_L8Uw^K92=l3q2^=tQ{CEUT zxAp3Luvp`~wp#t4#hpEMAgLb;p90Yu9gHg58=GL;nOd=yIuY61{i&o++~w{aut zCmF?p9fF5oO+uw)2^a>-x(uDAu<%H-4fac|3W>pE21{ZnAw1Bruf6xeOi*@?*EVtG z9rm};;dD;;a_eoKKAJdDZJS#TYDtj2Ke_rCrC6#gG`KmCJmMDh7*;R65M^`vS5DuU z+zAyYknn&c$%BV2zC4dDO(t;hzsooASKZU<9rnBJUF<~ z$9wzMfT!*;@v9VQ&(xpNs64> zG-aPlA-*~Py)~`W(t#fVL)#}`2b3`bz5wX?^up?#OwQV$R+3>nPM2wu!T0{GZ=_f? zf7mn1(Vagosh%pK*$zi~z$_Korez_xS=LcR_4cc~vioJ^RTE+G|*3GQ$h(Z1;JqLhL=IpSj85ZqcvWWw%jRleN`NQK_;9#xcvqcQNwoU)b!J8QICr2QO(-=g%VRjC(``U#RZbt2g zCUrzB{1k%9AI$d@0-OB6R9>4zM+CUNrVZlCBUyqow#|EmR{&0SrJ(J%ztz9AF%OFq z{0H3fNa-4;*_6@=Z$n>;H{X2i#;iej`zc&ZH_P!G>Qe{VwpiZfS>yI^XiP*8l+pmv znW>7wKa-;GZ6(RID@|%=>eA2zC(AZ53Q9%I66wM3TgD$bE#jI0=6LuWQ-$Th`gM>? zRo&(jy7LJC1j#sEnGan3H9#M*@l zw*cgd7@|bihOcB}XsJF##Pe$B>dtMFi~Jkekt8!EoOumN^Uoj^AARCwdn9`BuH&Qi zH~Bv+tH`Ocw2_6f#v$_MIK|Fjf?<@yS5p73f@zaEeCnA5H$JPNnVbin?EM;={Eo&`4q+K z2N^SVuthsWZX_}DH78v|axB@;%n&8-{j9g_p`;LX+>kOpfPFpcVMsJFKv_E=O~!!& z8b8p5sZg_Om0SUO#Yf;wUnC2-KjV?N1IkdNH`w9We6~<_m(c=9rY#=ti}M{L7+)qx zj@E^|x`nawUXOY)A4x1q9b*Ue^($^e{o?#OzXzErHP4TM$f3qlhMms(6u7rJWyK%0 z3Zk-{Skhu%G|T|BkY!%u4-x8<$Ytlme0u|t0d@ErnS!*TyKM)H8)zxku;iXWw?o;& zXatdwfYS%fN}eLU_b@$Ch~H2%_Y_zFVy<8tK56npq>RBRZYyOPB?)WeYH)SMlCE9n z4U%!(-4XbI3^xaKUa;TT@$DDq9|}LEE2sRu_)98I2<1$o*X=5)pRZ;GS{BKzOWZ*k zTrdO#Hxndn{Y9_a(#C+&BK&*P^ZfcL4V|!3N*boET!f}rr-x`N%I@Ah2~`w6nM7a{ z!LKvmMt8|k%;XyvK;c7fvZgSI|Gy{9kHXxvndB}sd_vN36KfvLr5#Z#`Yw<<7yA7F z^M}|=>Q2Z#R0@BAJY{^hU#}#QFifE47_PV&@S}`b#o5doVjnuA(gpx*8Oc$eGSIEt z2196e#luM~YNE%p_<%Hk3!W82;rN7U^n@X^(8-_5pPabA3OJG-DEpskTT&^w;_S%+ z*Wp`$J%0&H>VW~39~vnu&C42e!YJ}T!yeJ?F#N{dvi;GfaE?G{W<^RqU6%hCOIE@e zZX6SQ9`_0}>J>})#8}%9GM7J89PckW-6Zk1z{tVM9R!QkBj^p$eMA=D-)0QDS^DT6 zu$5{@9vydCTAJC{mpNCGe0SHw&ogX|X{@_~4?egA085+px<}Rbr%b=TYTe;T9Skq! z9Jlxv#>32T^!~2a7~vm0joF^dEBt%*+o(FzGVeQt#?WFh_RAeSZML`)al0_s6S{Oq zIhw`)8{+?S{w-Nh`a|={pAGdmjq9aEt%^ODf=g^gH(jsT{o@v4i@=BF(s9d}=DYQw zt7WMf(cxu{;%Reh2iHP}6d{ImWxY0={fb`*@tEi{SuP_U%jUv7l>UMiKj~8nKkzYM z2&5?Tgx;OTK#8Wt*LcP>31s^9WeE{M34&Ey=jHDQVox*}0EPHeeEe|P7Ilj=i3vhpCk z%z~bd(B^jPEN0~yNTFsZF{_=zL`w0d))MG%Li1y~TPC7aJQ@HM#*(p7Pbh%{@`hbwN7utb7#q z7yaMEUl0=ru~^ti<6A4xb(wrybWFlQqNe_@D_#YGF&0Mkd4D~QR35I*R{qPHG4pJQ zha$mTLprY2FV4hZT)WA#*vt?EKQsS?$b2i0Wvli?T{Q^amp53j)IFJrHI)ei?*n6B zyKMCbr3{g#_luZ9s=S;lqGO35c|W(jsh<&Wg!_la+rX=mLn*K$Oxx3Ea+TqF73EDd{FgJ<#sP4zi| zm$(~@ND=CpUJ&3`5pq*cNABwY??!%x+MTj;7q66B`feK^&6juW%~#OnO~h&yo36CM zPbGs>WS4Y%xwmLd!)!Sva<>^jdbB0Xu3f27V|Hhmb;Ya1*$AqJEnc)|(j=Bb7-`5A zvj@ylw0cpjY(wdp89M0rImz1}YJNBru@r^CjITfllCge{QKc&%?fv}58QQEX zLzXg1tTrRDCT9LiX$46I39r3wBblDqGES4yCKo6ndv~|<+9lH*D-tE>c!{bDZicfg z1}WJ;rYt%2ksG~%Lcl&qW4whtrF=(upDA9(9}-IQifX!0iIfv`%^Dc4@Xm3(Cgh2e zN*}yp1u+rm5CnO^-l7b=@ed{w@j?Ma1s3fWenouxe1hwdG!oS_a9CRDeHiB4iXX%afV{xGrCdL!+VP(;RI8u{rEIBA{@F5hlBzm&`lp$UIduDyjV zmQA=|6hSV*QSU6nRBqm{&~I_9@aUzXZP%Ro_a0RWfl*u9EbmS5aiCQ<{+c*icm;h} z|1VL_EyST*A-Dq2&^cW(?xk66yG+mAM`l&6Nechj;rAUk5@<*gXkY$d!R8?ZB$BDS z(0(xkRnrdS}u~ebmOW z^i2}d$Lxvi%&*J#7rHv*QDSUh%;x223AfJ~=2<%)T=0Z^foP$ae>=_O8wy*8E?o+h zY1HUD&I;nZznj!HWRkyscCva!;QhMT-`Hp( zCXhtK5ygKVU!4PJNWCoB#)>e;2?0(snAcfDto_2y0D)uvGw~9=?`%Nq{PBYcB%c2w zN76p)QI3moJQq7Mp#}0v0js|vn z)!)o?JeYl61PUJYp^@^o`hkKCvUH)Wht&}CZP28;0iDOEz+QD!SA(RDPszlnIg%LJ zWd-?w>i#lJMO*fH^%P?SjA?XDqbzsBh@SwzM0<`?FL4z07vFTTY&tpG`AB9cD_!`I zv9t`AiQI};`T?}py%V{W9^W`$m|+t&Y2&}Z+8?AY@|?FSN@uEQ{8kZ6v=(2S0w!l9 z&jTq(e7&R_9JF(EpD7;WaSH%ds)Cw$@9T-;6+Pqx2~zt;Bh`ovZH~mMfWGidA71M zo~@s8Me)0WQyJ=_qbK}Jv`9h9n~VXYS}^Os0Z^2achND#%`n(;5d=-COPnA5~C-z+RC#>e01NDmUJ9 z*^h7A^u0eFz5d6`4=on=5rX`#=KADo*w>1C3~^pysHvRqD5VO$T@k*u97zl|osx({ zB%aic8bODAv0uL2>jm4nZo(}o(h=(Z)Mcw(@q z=lZ$!^CjN5O~xp;gmUtFs>TU!nWX)qPn-{dhHf;$`o8`d{{)o{8*{&1XZ{`(>YSZ!s zl%bSJ46>OBD}*09ZqDDLhnMXNMLoI2*(2@3u~91ck`QXWM$%Xr04BP9!J13H;{lI#@7smq=K_y z?NV9iup+|XTD8@la-Rd525z!noLKlGfK9i4+z0!^{d3A8IWC$$rOEpPjB}KU8loWr z_U##+;&yq6)Km4MoK#MhLG1Lm)bNXSoo-bCq_SWyzF9-Dp-rE9dWt$#;GA`}niv3a zpx%fYe?e_LR0m2ZZ8P}PjQ7RU59$%9f3TO8@PgvH0hSR>N4S}Y>Qd`hC~%31fm?OKbN4v zFh?d0U0V|tuembCIfq)s{pOr1QT^27?B|pA$98i8#HDYphxj*lvU!#%DPysl*KQ0q z_bZXw?OdOzm;_$sy?^*~K4cygbJ69>-`_=zlK0%!e~G*q2s*ZLqHO&@47a1zZbx;_ zAu>$jjbgTB2y`TiWE&O6tCt%|jaF=a)%HQs*%e&4wE?wR#Uo&Hx+IbxHEdKSWHZei zVoQ6%bwewU{>u-fEmREs{8VG46U9!wv%+>jy{V^3*Vh4v#@#31S$>Zfh)}&QrVy2{ zSD7nV&|B`I-THHo(-g-?WBmgMx)&2Ystc@WU}E+Ml|=Z^XO`ax6sljKW5j3ZK{engAfXD*oxW~$5?z+TneFbsU+ z*wo`%O_QQAD1|OjeHX{@d8*LoZvgypiE=s|MzncT*kd@E6aU-+7H_{l4hpvY6xX~$ zA1ScOO{79C-qlGu5epgnOadrMX~0XO48TxH^S*mc!rZ^%xAIp z*?$yYhsIRde1Dn}PC5GfGq!E=jP_9Niig2PeoWCy>U{)h^ki(L2MOfvfwYebUCA;b zBuG~0fs(a`rMiH-IuFrVn%gM(;Oa+=2PG0Q78PL>z7fmC0KqajlhhNA>5yDg3F0Qv z$O>AoezaIBmtUpXGm`yltz9cn>STUjBw6SWiL1oO4z(DkGcfHl@d&#QGS&6A(nVd5 zb@@<_6BT*E;esp|neE$mUAa`?#vXAFAPbwN^#yi|f_$M#F69X~w8kvP2|lx1@ZpE+ zi^&;3a9x z69!iY9z(}oeI6AUvA8b%1$TRFi`jhtWdV%3HF4DM)ibv6}9b(keOeBWJQGvo_h(9LZh2o?P znUOZCIG=Q6T2nQ7)YHMKU@E5D=mZSWcN^6fbK1c2=BP_;`_lUVi}|!Dxu=~cJaKy8V>*(5 zF1J4_{HCUU#oISyeACSWnGR9|9m9_Ef0cdQP}qSX_=%$ z1GN!g(v}FfeFhoo%I)4PJgN|l;l$G6@3qP8B&nymRN}VWn$EF zO5QxXa^Bgt?2I)-KnNYO*|~chL_{fd;NBsM5+7INr7(hHV{l_PWym=WRZOC+a8TQ? zX&s)4AN~VpSZ_%1Z75?g1$HT!tlsBaPVcKL^V8`@)G&@#G{tO?`5ad9?6bd3_tuN= z5jh#}x77FA`oJ^6Y+w*s;ret(_MrZ3uW0jhrX~hbQUf}ZBQMSUaJ{}aQKMDhZp*wE zj~PwxR%dMZ0Sgo+vf`d;BulQGl5t6D%BVYj3!DukzxTNEoHeMvVK3_7%e z%p@-`qVbYm?Tgb21amI3o5P~2i8lWbE;4>M|E7Okpfv0?SH;U%kWof&eu>LyDgDrn z`+ZRNiIpOn9&g=*xn0EH5-tqekb-oF(Z6(P09|*NZ))*)*{alQxLI`CELG1OC^1u~ z;=$K9hWYet)krWznff-*d;ZB~vUkl5a+G%hN9Th4Wh2X{y64a&|E6|oW&&usu>+T(?6uw8ZvR-U-ybpxVaK@udcNeL~FN%u`hbP|p(>0Y~jTG*2=TqHW!dTgI`dpU;pHK;FiZIHt40uzO62B(OZ2+hqQ zGmFE=lT)IR9Awyy*1^O)SNEKL3>VBi9_ZRzu+N_P162p>;YL7wH_eKBW+oH{hy}7i zhXwvHAdr&DFzyhg#98o%EP%=4vlNr%z)a6kN;0Z1X(y5ML#!RPl_k4Q_wLfy441;q zteA_N*TV!`w;&;Brx`(`H51v9-So(M!Ul<;IcW6v%_~k00#Ek{} zRI-_LyRG-A0})tpb`_YbAicur$Ep)0c3`K>75~?>UQ&w@!2%bAzs(-}phEG^5{KV7 zzmz7ET;sGAPKx#}i@(*Ke#1_Uhz`P%3zXYFv<2Y&23G5S<=^^VwibpleVMNYoBphe z5My&B{E8(^qmM%7-^J>=InKy=KW}itD%p7xV%t6wOlRfuvQ~W|Ay({CpaAl@vb5k@ z(cOmBA$CY|_4*KOlp0NQl}je-(fOJ4!>F2;(Q2bXDixFCm#Inp9i1^2hTE_)0z< zAVu?|BOsuA6BZ8{LQ3v3%$^b%U5a9%h9cgp(i-vS;ku*YBn4th1uqMkIjHQgzldzw z@7gp% zqA!|4bjBl6P-dB*;~VRNOdWsT$O=Wn;Ys)GPliLWWdrZM&qkM|v?w@fn|;z0$+8?M zZD7yf^1T#w3ORqb*E{u2O%N~3Ls(AM6eA^P#a40CW!qh-_rNj$(0Tcgd!+uyc6!?FXT*_O?lN)M;$X=3vzS_uLt~w?&DWRx$+U0*r}E%DY^)oULA)` zDhrOpVes)@hV4p?4nZenGz9njq!kUin22N{J2=0f!u_iR?An5+HR7sqY0|m=SI6j^ zBTW>MSI$UIH$Nn?!We89y&v(k^k*mtr?wqtw$5=Vn+YeL7IDzV*~~45@{jRx|D7b0 zpu+mc12><2-!;ZK>y6QX6^G?_Vjk?h@V3rJ@_s?q|Z@9;R-5>mgks`LjkF`rx| z!@gbiyN?9}anIfdjx;ROAKQdgLmZYv^e;k|$~eL_AO{ISj-T{gTc{HVVH3Fq8d;i< zd-6F?M7Hb|y~p1BWO^?d=rePUMtLR_O%o)d4;1U zKYbN*F-_-ru6jw0J`?ODzkwftVxm*G4`};43=-)etXOZKxLFO^oL_&{g@0SoWPUnv z)PU>~aLd5DC_#^45JyV5=&~Br#U0Iz8N0VdzTDoIQnuro1P8@ zBYNExkS5cd0*pqu33adbO*W|n0s_YxX^O{5j8hKoFGAT+Wm`#TFK}Ai&*}N5F(m?I zHQU0e4?DeC?7cG_70S2U9(L~1X4<}?c?NT^F2xszBI4?Y^h&5FT0^vTTnEF>ImZw) z-Z4$0r`A@eENYJyJ2PHHh|Y_+&LHd*Ew@I3yP-cDgVLg4hxgmJ%n7Yr;){!7uFn$D zp3ox7X%_MQ3Um8PX=>j15)+Hc=i5=9<@>72J8BM1@eCfjZ_eX9jVHJRPa`kOe&hCt zl4`Qv{6?GZ7J@$s!zCw7d1+D6)wfz!Gm4g0it~-OY@tr{cAzCHVFxojI42YRE%J%V zz5Uz4P&Uj^U(EQOxQqk!*M`AhhbGNg3-@faq~-c;SZ#gd_CY0tBK(&emsxvWb2$W| zXAPuYm}WtIFW+k_{Bp;OfL+k9;QimL)vJ8}JQ@um>1uB218sHZlW&x8p5B#l+{QSRSV4LN_;k8Mn+P|_xObQ=aKb030iT8Oq@R8r z!xySQqt%zBhDp-opP_7AZ9DWSTK{xdp;eG|x?Gqgq|o`~l~${GVw5%G#@w^1`74)1 zt~V{0jekc@KN;apVD1@X5o*z0^H z+`Y2R8gc~o+nGeNY&CC@BQj^5>5ut!`Urbs>+Q}<3o3B?GwBHrKe9D+3d`&hZZ;)0 z(0)mVbEna^4D!P~r!j~u*{QqJJVrAEo`ZACa4ga|t? z(Q(hOw0JnsDAzs(8f3z~oqa?57@vsO*g_8H3`3{w!@u6K6L$WpOc8-wr!{2S@=X;G zP4X{wp4p{FiYin6J7>&I#1?x;KJ~2E;a=n-Lu=+a{7U9_Ybv_bll74o&KN1|1Z>X! zXjO!=-?lURTu4nR$ydLBsANf8ePRlev1YeDh(56TW$-tCHElwd-LiKH{Ma`jD0n%Y zuU;SwH$IMKdWyH2jx5Ri|8ez}VNrJ9`!^s`(jg!e4bsxxIYT1? z0uoAhBTAzx-0suRMIiG1tt#_Fj9f^ZcB=B-DM;0q`yx2(Ohv zUn*s_&G?snxvFR00(jmeM#6V?DdVZgNd@202|OPw;WCV5wj>Ih#7Bm++M7j+Yn-8#%A>T%>TABjrphZN2E8m#?N~jtvRL+b8*cr2Fje zO0=?jmsHwd&Y0SpV=P#8!>-L_LCUX)g1`UlaHnTY_wVgiQSQCM7i8=uM?W!>mU=f#)8Q-ciJA(xVjPZzk>y>^{Rt%S@* z4fuPH1#}tPuG#lRXO!!D)pT-WY0FXrt{*ZMn{%Yq9*n8A zhrskir3R1bDSKe9SvOacD2aC!+|4U&%C`b+11W9$t9IoA-kA7nOhx8LE1S)FnJuoU z@h_TqQ<8juiJp8I!(Dt8ou+;y@+QTzHc{CHYc-B0&!Ib8U97hif1mp-|L#z!^gzeF zpt;evW+YX%{0YzSeqe?20`o4%8~m9<(Ml{t468X$Ow_|FjiDE#d^1dh<-z^8uvZIT zHrXvk_SoL@lZsNeUG=Xwcg2Lo%v#)K_JIVG40t)@BVa-T3^GFA53ii=Y0tczdUCdl zSWZNV-esoej1YXIK8Qk3tEh9+N~E!?}mqz8jg= z--^e@3}cjxHPz(sV% zIq7}1@xS)Mc`u~ihF+U6v@B4HG6+y#$Qy6mF;`M)XXk%0p~7Z0sSpUY_KTf{it*f) z-F%wtOnnw5Dqzs|v^If`g(UwtM#=0;9kb)6Rj)>w!p`9FEA-pEP93;Axl?-m&I4-j zdMW+()t+?S%D}ik>$0F6af`_%unXOlN+rFdha^q;2c_Fs&`aLn zly?N<$4P@OY11H1QJm9}8J>A_-6cat)gl#w<-MIS&2?sk5KgKqBmUmB0rjOTsed!SE%?@Z~KID7)JvzBCi}6!7@sEcC%9A#3-+UH31Q=N0kt@dNZY-$r#vmix z2^V~1n;{S5p7>Ym3ZnX6IkfyZZV>-sNMYgqnDJUN=7k{#DitV zTUy(KW>)|O`s z#~js`_aPU_nJT-A1iC{>`P&HpBr$~wTm#JyG6kOF3Y1nV zs|rKN3mzh2`Tou%a)X|vW@S5V6&S*{uUgpHZYx0Ck~RnuCh5*fi9POY=svuywhX z&lvHqcqGB;S;Ql=MbcOhlXW4Rmn5CwySIdq`SknX)8DeKB`!{_uYVtDY>;c` zgiZJ;A-ZsdmvL9hh7QYf5u_$z+zu|#ard0)eV=iGw^{!L9Fr?P=yHUU_N#CN!8Q{>Qyf3UlBK9yyz z?Xb7PQs>M2QntrtDJ2eg49F-V(dD4Jx8*jHJmpbp!8+riw(Gj`%UX=bTB>v#{mP}? zNBnl++GPN-GxjVCR`(=Lwy}D-7hQ+9%5JO7b^Bf~*YGi8^++w76VK=_Q};`aGiG!y z#Ybo{=koNKYie~3S8i99qo1ZIUKP9b#Rq&)hMhYCtfmRj8Pm=mQzwE+RPpN7`7V6 zDYLnNdqquC9;+r!ucW8b-Vi<$dOhDCOFYYc=gPpNMr%?HlS8wUCy;vIH(wfli$xAz zB0r0sFp5`CmkIs~%chLpBz=Zrfzla${2g}^^nm*d0#81IVAQ!pAqlOxE4ro|HpzSJvVar( z(xaR?{?6Tv91>FA9Z}f2LN4@YuulP^QI5Wbqpq$}7MV_`Ij#jAyy0S3b=WS~ZRG4U zfKJlz=98RI4KD$Q_P|)y_#&t*Uq288w^5Yt(jzj6hM|AR%qvoNeNLd8s9e?dVEeF zP1~l)2s2AsBNltC21is5emT`R$f6%=Qf9=ae6erQeEw#~o&jBA=8B(Xf3nS9Dc3sA zw_r_tfLiy0Zt^xKh80A{GD@^y_9EEow+UQ1gIUMZiq_Oi7gK=PJOERHHGdfHT~6tC z^j5{HcIS=RNb@V$>X}>laBlt{>zTD<3Im=$HvJ28_SWyc)PUtmz_jPO!se|~n^F^{ zY-ySxW6uFtU$(leudC)=Odf(m~8_6wIAJezyW zYDMMt6vigNu(`1}26}2gzQ}2AeA8?SSnZCce(}EYkx`7eLmW?GLQvA=FCX*0mc?>Q z4sDgF>;v=VB1^;_M!wO0Ah3PvS2_v+i`;Dk^YJ{mh2^Y%#{T`s{QP@^=egRFe$nbc zY1i(;YD%Qnp^_hdSA_)*>q3Q#>I?0QyE1Ip(X0$hxnwmmetm(`2-eXZ-pxHDvgeB? z4RygJnEl)RApPzvi^|-Kdh)3rvWY?E%_uS??P#UF8 zB^pB~%)5vJ>Xwkw&u@F}v1{3Mz!sY=R*IQB#(YeZV)>#e+{XAaTqbDiB4sf*`O{@J z&#%JRPq=sOlQzQxB^rm(*w3>I=*E3}osHbZ94&}-(kUrt6s0o#{_6Yk#)(Uq`Em5} zC(U!;#IaEC3l|Q2aT!fF!+OZfyLTb!GfB&V1MOe>D4@q&yj(KpOthg)UnJxh9G5GV zo<8Vn$+&JvO%#t)ubpHuA*eYH^E%BKRjUME~}TL#NMw`}M`_;fpNu1(gk3KfjgqvO;3}-Na~D#{Owb`5_K5x!f|& z5yo*&Yz=BDRb@bLu#40-12hmpuFEtKH@E?PsO%{v*dzT{^X8%-C2$pQm!Bs;6XY~KS3(;^;Zm{L7vi(VrcqA5TwLr5z8x*+P)09 zI#&>(hHlhMuNy(7^$aeEi?~DImMugO&^*K8Wt%iVb)WF#veDof+(&jL%U|5>yz*^g*X4drP z=^j|P?K^hr$m?a`N^jY(f^%R4j3&jd@20N)wv0V-lz%po+{3z`W~t6W_&7d17-&#Z zmlkagjfB;hVVhCIqZrh_GcnEM-_iw~Ua>!&<0;OuTbHM#%sdcsgi=eF-(7npRs8Ci zApxz;VyX*oLC>mUr)>j!E6+ig;E`qAPX=l(Ewbn{XShfDQL)|>PmYn0kMo6!+;HFPet6+3GBLG!=yn zX_VOkHR8;&H><#cHl6~F#aA&3B>mJk2K3%r!|6|kGjC-+19Y2LUlQOIv0{%a%&bY7 zlTxQ+9Sh%CI1ROL@|p3RWt8KW&WW547@xnZ?69Ni6D3<6PwFk5V1>7?Krh#yuUFnG zR7@_B=V~0}+`wTWT^P(&9Qt;0t`KA&RkPkOL*lwI-9Zs zD`>qj{6j@U>Q+`r$`d`1X8qQ_yY=b*kZuy`>i|+h2j*@uV2XsE2|&)Xqc1~P>LCsTvC#40U%niec%sSz#`+%L*_s(x zNXkigQ{r|>Ri2%`{+K`GGkc=he0*?&u03IZ7m>)fW{T+l{G~0FTIPjWKf{0wBway8 z)HOe;Q7|Ff(<8-aOvd6c`+2V*N(W!v-Jo1-XpPRQ4*BKp7E0AvF);(#M4PCejI>nG*A)Vv4a`g8?4tx)4)p^w0e+Q`HjlKfX*VTBB zmn}d6bt#LdNvcydM{Rv6Qfy9N{MvQ)lE$UPScP@8LGnmlP$%ImXSrse%{!tI?!Epf zd$l9D&N@d_?#jrOKwxi$wqfscUzCVVjMDhqxAACCy;v`vn}^{)p-l|E>+PH|+@q2x z&AQ?$cCFVs;v=DuH-W>ak5(;f%DK3c$wsPsmkCd%T5~kjDRFv9g5KjcI{ zXVFAUuy$N;w>Hw|2&TPU)(RBZ-5kBgjWj=6udEwaZs886HHK(gO84)#TShgeHj+AT zEj^-o>F)cwz6biAN_s$4+R4nM7)R(1hNusS_Q4UNz<}Y>dn4h$exPI)`QA-McJ}iw znUKB=K7D(9Oqj6H_o)84h@rFFkkf1gz1xNM2O@=gODk6?4>boklPd%IT_>+_em0_+C-~069VXdl~ z6mCGzL6dK+#&UC8i>0!wZa=;5M4HDMtD8Jp{iAGA~nf$p2zevd5|AR<_Kt(32|(VG9}+} z>_#uIgt0e}8;no7RV~t&vr!OcD4t4wo}>Ct$jU%-T(%`LI2wTyCM?eh3zR>|xSzubt6+(C zirFzUzP~ZltumSM!zJy{_(0iHaisHh!JOT?FUJYKKTAGE>3oyaYBVfBf&O$WY%{s% z*rjCh-s9+t9%nC(AsAB8t%0>US5+)+@4w%OT00={#dbKY`fC3pH!78|%n=Uu1fyE! zz7s!}G(+O#-_517BRKJ$+X8wo=PO=){aKh?=C}1Bmd9|4Rm`<25$W#4u}#{mOem*l z$uiXXDF0ZX*93xGcE}9M!S5C*B*%_Tk`Y&!C}UA9e5d5Pk!dBv8bpGDjIt z2@c1VyNV>Zrold)B6n6e=0PDBypOahFxrSgAU{ZlfzYY6HdjjA;^{Z4;^IN@7!oZNj8~Q&B_9>}??1|yn*C0b!>i%JbLELNduwFbpaMRWpnkSPp8}ET3@P zd1G>|&G|Sq&er-?sh9~;-a2Nm_s`gc{-EB1IOI|Ts`H>QK|D6({RTGOqk82oc0y5} z3U-oM|1qgK2{xJ~_|_odYInCXGhO=d7D?=HCpW^u`*VrB?Uo(A91mCpjehMp-hL|p zH}o6wAo|yBoF_~IVVeVC_hs3uU~HAfkFEWZJHa_M+5J)Uw-4=iRpmE(RhhO)ee0N@Pos%g1pB+PEb`&h9fn|lSutSD*a=!u&a&^Pv+E&;H_~mSzzWrhtUzVW9+e~QpMx(%2-Pl6N1cwGN8aE%oE|ga_-3dfhnTBOlC3(Bk9`C=CPDA-4x9|&YL@r~0Lyo4 zqFkl6KY0COJ0BLwClS+ZE`)d}vdy-7>^xh?yj;`urDhb;v>UI#ahQZ7xpb->g!wK_ zmhkmc4J0Q?8gM&neSh8TnfZ9+V$ zebe(|mBQt_)C^;(OR#}%HV!fPG_r@X zr3T1Xvyhg=LD&dKdMq*o5|_nGIT&Ca+om-Mp@Y-iok_Cp;hGnz;+b+iuKjZUoDYn_d53QSP`%=i2TlN;eQMV=1;_d+z1WJHC1|LY=t|gQYQwm1v(WR;Iz3VOF!_JD$L z{c7JDPH~m^0-=@4-zBnISd28!WHa^B|7!xWxs8)Fic?P)6$X`$c)a<|XRqlcl^5Ci z@jRuYuP_F{xYcNM_4rA2NtHOBDMHZU;l@KLR+=P-e0IaFtM2=E4HJ#bhH{C6K*D>K zhB26cjeyRNZ93r0I^wWu+e-mR#Vg`X%B#HQsjj?>BC2qb+?zljG1w8r4<$B_fRWB${f* zcU>;JQ(w=Qu4;^ky?=)*@C>vqu@2*L-T`6bJl%%?QWr&=W@ndU#ktM&M4rCg{mNL= z#P3>|RJ+Fnj7NZ*Ts3ZTeGUv&XJ&3ZOT1(z#$)NGKBX+B&<`0U@)hK(Uqvd~fep6T z?4O}aFKIk+DyBj}Dtb_E0f}pl?>6{q2DX@LtkuoMkH#K$csSAIs+@2f=LBQM{Q~hh zW_UrKeCC^rsI;uptS95bMg9beJG>hK12#6p={5`F`~0nzW%+(W9O8vyW3GQyG~~g9 zkRbg%YB|dT{0))F`GI>@tIC42qR;h6rLu(z3q|Lc!Kp(EZsi zauqOgy&rCL0_XffoIJnMw))x01;&qsuV%QK;Lkiq7{|WoQe|f@P@5ER9K;_LRX}K%bd zv-7lxmr0=h8bhCBlzrmd?W;Jhof+KBo*Q4suA&|>!(%U%4^F`Rj~eSq0JT@P1*0HT zVGN;?C5L6((1k@~NulKjJC`xd3ODkL^P zClEu?9+Eh_W8cb;oW3jp#6VZ7j{&hANxe?qEP~ zZbp&z|5T#q7wmR%2u2oPv*fdCFOm$-(dZ=Mzc9PSA*zu5Q+%MQx;V*EV^R!`Y2f|<`yCI6ntt(NL&3$`%z60V3JE}lY+n-^63{HXZ(9Y7JRCdhu< z7+~ugEB{_{fBvYlsolQ7-QYDHG5tIaw^76JHv$nc<|m+`ZlEM3CB2{ZE~sjb8Q04@ zDakB6if;*WC%+pVt~u`dU4HI|DFlx%f!w|C$Bq^va+KF>SO6Y1ejb}?lgQMXf0*N9 zS;$*T#jSNQFr-Y_Q;3zRn$ua*Gey_P(~^5(TCyAVj;va#X07BPG>ldFw09*;9qBpe zNV&khb^%1arjJOi1`Eg-tp8LTN-xK;lo*9alWxZ|-E98d6bOHGJmN+Fd}nR6c&^ua zzMjNHe^1h_heH1`;;(!LLuzQ>*Rsux6E zR3p^XUI~X&^r<8)hCb!>(}Rt%PX7P}lyT8bTfF8RMLs%n5I#WB#8mI4KaMBXb5oEa z*ZDy+z(mN=loz2ao4<4Vk~Nv?VYiFR=VZ+qv73}U7Jf#I0;S(?BSINX>dFeAQgMml zY$}gfdA}%{5U3>Wmp!1*Wg=rTTkvl@(_5^B#QuBsfg8+nCyq>r1HfW<^4!#|Q?i_Y zG;D0dy|%^L3pNz)M%_6*YE03Sl2VeewT;mXqqSCNuCgk*Q;N+?A3<{D^WiXZcTa$u zq}j8LI}=VZf&8AjLxVoSLE@m8GzU;uJ2kadB%wc>+%`vlMf_M7K7s~M(h@~T`JW^2 zM!2*3v;+H9VJZi#1F@ErPMjZ(!w@AQL6mfM6z{6dBoM{-%yn!%_JQlg!AE@UutJE6 zo`38h{`uUXNrGlwCEzzbdsp6|YFwWCerlJTMHu3qzma~BZqnT0e=*ztNzRlO7Inac zhQ6s3+w8OYjb}sFpk$Y}7=S4}UE`9DnWsDaeYvo+U@hp3G+OC^Yy4SpX#FjNYm^Nq z>u&!{`q1C$y$U;{J#_ktq0RVe<$QU5jhpPAv{uR7Z7-<>VX3u>^patj<`aDBEm3Up zma#3GF=ECi{-JHL4T9zNNepQ=n9qMc4ls%_NcbqVN4x$u+4G0rukws2qd;kse}GZ! z^ysDya5rmp3^8R)iOwz!3m9I&*_%w(&Sb&;=DuMX!QSCS?H(IM!+mOg=)nS3$ydk{lftH2BaaWECKW8vc!OxsFCGm@uJVW zREjHw>C!*mJk+dpgbqEbmb-C&LQ$JPFy@Xy$63D5hg+K$ys&#A5P}Wu^b6|t}TPy za-$@=U|LhcEObTxe=RYQ3DABt`1!;G@j_Yt=h=puiChpwWBq0FLl%q_ z;}GR&xlc*TX@}p(7%VF*xprr|1jh4_c|W;%>?W7?cq#2#xzr{};lW8wFk-|7=|oDG zDu}JxZC<~0)j)(!c-J&tx%s;i9yqK#89Q4&U%_^YQxAgkrStiG7--%|-Z1!kww%ja zEC7J^+zv)1m9lr?qqE=}M59pcHYl%p^hCn?PsJ^a{Fe!>~zum|=rowS0Mg!hO*PU$!+4HGU{T=6CjK>uUHwjsKjs zeXIaJefI5*i3Ikm73fD~-9J}MnPXxZ#AYxyeO1l|Y==pyr>QupR_!f!O7*xyH0evI zb_lSUmz)9Q#x2V>U9Jt~E?&zvDV5#<6HD586IL!e6EIQ_5O#gg%j&Nw*3-ruzLKM1d2(Hl{59_D?c|Ed-lH&_rk%kak) z^C8jr1H*!(kZi4Vy=d>G!Xx{U1M!kZse(@moUc`qSu=sBXnGzPvZGty&r1hmWwc7o zkIvuF^dz$mB;Q0}GBsVD!J2RNDQ~>VOhg6UYk+8(D@I*2R%OgK8_J_M!?^;vKI^Ia zdhOgk@4rE1?%ZoauQ&pAqwfBD4!Ug-IZvdzMwfyHIW9BtSp%ozyeJIQ6j)NRLiQ^I zuip8ecGFQ#6q|Jh7!Xl0tHSl(ipa~o&waev!*Cc*8)K&t`raazJ54qKpH6%-n~{v) zu0A+uQBcE+9-;=-enrAf#d2ps`09_zAIy>8Ylp1}@)oMlb#q3e?=FkQxls*LC54Mh z>COhg>(Hf{UWWLU#C2{)G4}hp_`RbDB}eY7^-iu5Pp{W~RwvS?F;M4w>$R30$o-9z zH-`CK9f~bN;z}GJ^Sk-GkUvo%Q_kmDXeASwJeP9V+5|w3hr>)@|MWC~9Yqr-$=9Ke z&Z{n%3{wqt1&mG^{)StCf=MrUEiL>?_BO8Xl?giYVCLt~o^;96vcr7z9e@BDM8(FT zA78;4af$=eIU{k}m^q!C0;RN7U4oT{7c?orG7_@Bmbyy>@<56-EHdp=eFdZ85jY5< zwO6<9bsKM_zrlZedUVzN>+|t(^!w(`9DAOYEk?EHW`Jz!F@u}pVfTTy)~BI=jW}9v zQ9479o?de`0S77<4!41|ihT2sLF&AgaJe`BHA4{9r^hmXN@YtZns#*KuC@}L)sgiY#0yw~tW(u| zd)VY;5`gpnX*bQOlFMyeyA*O}ovL?j{U->@Q$@&4vBd17ROZAX)J4uvt>gHF+2^N{ zL9PVTi&=enPyhayQA+AJwur=QI+c6xkN?u5P!cO4HcW`#0|q$_f>f5MDCX_>Zv;$_ z-c;Fe7WwQ%Hi(b~YrYZ>=W2?`EY`9crV*3>I@}20+_N2xO&%~t%^ZoZ{u)1{KV%3# zdWc90sqlLvtpy};_{V}9Ll2f8+N)y%l)EJMPI>x0dHlmqFMu#w;K$R$Oa?7R%5)oHY=7L3)K%>VxBa_CiMi?J0Z1pT1**GRhN&zT+9W0Ft4 zMduls)V|3+Qo;;+^62WD5Ni~?*RCU5IR`MWIEk~`)7^XUr^QOnO14lJIzWs*WTdI z)D9}>6I&^}b8Zjw^?tM-d@NnMPNJc1sLeo$|hhyr&5r4OgF8SrCK>wnuB)ok1o2TP+wW^gkfH)ig74T=yZY4PpFZa8(oIOnW@t`|)7LTjU{0KuJ zt>*zANT+Tbk<8#?e>$AOUj&E)zo;st`UjmBTAC|wN^iz~&Z;%;bD6u+>MZO04vH_R zWHgoOp+5B$E&^j2&6}+hi+{UOKAkHu$SvymSl1u}{{qwGq>+G#N;;3pFvw7{$?@LE z43P|%S%U`Y3;FaN+l5g$gM#$Qk$2GD>Tc2%FOw1}3{+mEID2&dl-&*VcI&JSiRGqX z>on;B^?d7p@8PquXx8^#L-zsJaARDFp?>1BEu+DHhUZw3hljUym&J?_9$jhb8lB0G z_-)^>4Z)6vloj!pzhA=7G|#x3O#Cv(9kI>PynB0e#r^R}gYMyD^^!Ug)L7+#pZ4g- z$Iu9o6qSLV^}Hia@8yh^E(X`z`YmHfNylc%6QMagvDP+4;_kdR@BWz!+#x{6xPwJV zYZFbDe*-t*x%c#)^2&ffR)EszppG!XJkA<@LiTz^1tS zmO0B_GnOfcsX2lpX}x+cNzRUqE%S`_rL=KqK{F7Qv^orYq>hHwsPbZ;I4LOzgmwuK zBT$J(AjdI$ZYp4pt@&dmMf15u+!oCGy2dDdC;69{S_{LMW%A|hZ8&L%3@zsvWu#br zuPj=sTDB`-%0I((QNo&01Lul^N7B1$cfx79Ou&XtM|(PZNyHZV*sd6L#imV{+;m$i4up(waU$1FEZMq+1p;X={hH` zdOWwCf|2dtwjE|l5M?YznAO3D7tGv7?)}P%a*7SGUi=N@Z zp^%HIcEQBKfb#bW+M)Qvd&`D=nA_Jxf-(vr<;Qhmxg|J{wn+5vI z6FT>Peo^uZpxk{*b3CREO3AWfWAra)87a<^k*BAv#1F~3Ybjl;?x4QO^K50==pfcDT2HceP`R-*1`1PLVW~mfJPP1g^x48N(b5(T6H(16MWi>eK8T z>_H{}+#ZA^FuWGMi&4BRd^l{O)vK}4ygrmG3(pcm1#)vPhA9aVqTz9yGjKk%dxfxc z4Ate60OKG%6;94l6+R;E0$4wt&(>?<7*3TTi0(BjVZ^)=U^moJgOHLl0Gt!RxK|Qs z94;!E4`~?We8*fW{!TsiHn)SM1G}O(@!{OMs>A0Z4ZZd=Jz|RJ;lfyP<{08;-F)cO zog3&CzS%C(+2tY@RnM5PXx1h^nf~dV{m1nbBE>i+<=^6CL&RM|`lpwtMq=3~^h9Of zIJ$K|^!t7^_EOumY?{H(d_Xp;RaJpdMc|(=0-YlVUeIvIz^ITl82f1T%`7%@@k ztxJa~3HxYv$+6*o*Ee*QtbGNGkYWp8dIu5f7lY{h@I9l^DqN=b%|Lf8=Vs$znEYo8 zBo9_p3~IARdeDRHdIO7SmSF&}rQIO4M+PWeD0V{{K)9l!$o2`AA{xR;gE~-u5JCfD zgVt1aD~~xbN*R}P0{6T#?A9qN;=IKEs}=qOEZdl40#6$DQ3dk%&eXf~Gk~8^&z15Y z2E)iKQGI0{o7-ltDz#n0a~V6yO6@tClzrSiWA`T-s%)3N((8@z`N3A#uM zT1lTeU5k=~BK=^Zl zXkn0v{NJa2()iI6(To78UC<1E{Na$B+g3N~YVzlt>>HAvUil_#48wwZNmSqDcVAzQ z5PMH@$jNy~Y%_m;$haz`!$&`!M4K`nTmEZyC@$4)zrX8L60h=RpeZB}(K(i8GF`Xm zrd8YC-^A0|dnep$n)KhMxy&m@nSc$@#LPg+-0yZ?9&UgfNg-53RVqpOXP!3`4wTR} z#%DUEkGQn|;A9g1d8mS-xsw(jyUGULf19QMu?q)bLCRg=0y?H@1|k~gvRwx`ihyJ) zB3sP%>nbA9G8z4b`Vvw=J@=KIDKN>H-KlN-rwWtm0PkZ}c11CP} zeRnk3NW-?Ra?2q92fICt;eUSI>y#o8NNA>;MubLWkce2VH4mkyrw z`&T;f&+C-e4E+R}D{h;wys}V+{W|-Z(KaytDFUs#jbXttA%NvqxtX949%a}HXcpnh z8UYvY%518UkTCK zjpjV9Cx(YC4J|1#GR)eYEQ~aeRyV89HS|1sZS_YDi$cG&$Si}wC_fOXm)n9^S!yxY zLIVJy8ooMNa?W|Niu+kF@Zy8p7Ng~S_lqW3%D-mtg9r!stI6o*hC8OgW{7u9&w;smD`i%$6LOYVym&ANgx3Pcc}X zD3IyNrmz(*zwq96lwwxR?gu@{!W2thl*xaoxq;Blb z!w3nZs&YQSKWN$pBSmkZL?a+;hv8grYX#PjgvKvitomYeh=S;tmu@cn+Ds4rdRm(q zmyszZ;8B+Iy3O0OIUI}s?^*Q(^G6cf*z%{>@&q~sOp0+;uoYATp?zaWOai%#g(&WW zhcc*G-9qvv&>~rS1|#t@?p~TqC+mCV`mTr zjELd9U8cJ?4Sz^{_CN`b2f)+nG6Mb?!~zWCZS;Qb?1E`{b{nwgL>j>IziB1o_~v5{ zaFpCmd|zk#`FjL~zA=ACP{p53J1-NmoD;g&{Frw^&!Wn7G^OF;|Jfo+Ffd~A&OBp~ z1uyxYgow%X7c{$VX2tTf^p~-XKSL4LG>D0J^s{6MD|q2iMy~=pGLP*?W&7E3|Lo`R z&P!v~ZSfdC8!tG~RCdUlC-e(Lh)~2kmz~dABHRkM9kXHPqp5PS{kcro3u0%o)A1Ln z>mNyam9ILoeIPXg z=4h^fyOMujlKMCqz$|b$ns=*CoBP(N5iQF=@NcvFzx|IKJr*q|@eiY(au;mqX8^M1 znDYumebKN^sNOrg;lz$yPTS0W+OcFwCaBHn3M0>L%NtJeyO#_mgbqFvDn-TpraW&+ zb>DoTYh^$}#4s-Kwr5={+lL=RpZJ4?_aphzzu|k0aPcys9N>!D+)ZiG`THH`5t7Jo z)@kKRJ-ZJVV?b3G4{4tz5%>d($?$CcqdYw-bP@DD*6v&TuUY=?59*lNq1`{XKkvK^ zRmGp4+MekB#~kC?QfNGg!KB1(Lxx zXE$QFhSTK;6H*P0N4o|X8N_8-*Tt1JySNF(`|q}Nld{msdkrQAjfgevwG>tM$M8T& zG7A*+QgCEzFrf_R9Z4Z-d$oyQcM)oLqcb{%|0fTbl)_l+18tXNl!nICXPf17i-*12 zl&*ee_0zt$4mR-OF^!;B+yh3hnbp6qAwI^Q*cL78w>MA0gAHX)c>do9(+ERjo&ab# z2dV4Rr*5D%ngx_ynZQ|_IU+najPezF%9Vd9;^u@+&Vml&h(i9M5&VF0bJ?17KzQA& zQ}C6h)K_-~(Fg1Du{Z1c5B2oiIs-o@s{FxE_mWf{fmvK_baf@^lb!n&roJRMbF&)?@D1rm8cFZ=CJGx{Vo04u+zb;tXwZnsA?sKury~|A8PMCkFWvwrpCo|5X07eRDLB5gen~ zVAm_#nMp3p$N< zY|~go^bQbwl%D|=5&|NgtW5#JyE{n+&FNEPXfc}o;EQl8$b z{*d?q$N1ro&+RBgs|)e-X1xWsYEz#E<#wS*bD8uQRWrFDVz8W;n!Vlvzvd|K z7T*LMW28Av^cSSoe~t4JB}`QF#Vs7k8vxo9+|*59EMhZ5navKiNhvxb#M=seR81`7 zUJ`57yOyh?_M~8Oxja)QEDz!41fH6@ELB%4YglF}i#*zxfc1-4Vi$n4Ri1r&>FU_Y zw>tMa5ATCD7+rEcRv^ZV@Vo?PH6XWiR@~8cM@6ocJ}jR7MdvZ8{R{+L-u#s$szMb`H3hTN2i|aa{7xX#xKEl!TL)f0W*N z*<%chGaD#QnLlh`8MHbCi(@3t0M2B{Mwraw6F8*rWE+DhjN`u-fFd8Nds(%SD= zB-3T+Ha5;oi2(OgGW$)++!mYeuk?pf>K}E}W2s=Zivwa6fJ16^;`?Oa2m`A8N;y>O z_vMk*j6o^h9-zC*|E>m-B*o}1RrK8{-yo#1oW=Cl{8*lEx4F&@2HZ>rTDb6H^eS)) zqfp-ghv#dRx2uh3MK7Id`W7vdHBzV?5Vc9(4`@H3W)3eSiMNk|4q-fwMr4#0B##RF z{D@*o;OnDxi!%UKmVno!^Zy<`4~IPXPe`#-=auLE=~Hm0{GJmh-w%BOAcMT~szGRIL+piBcT{8~`%>i(p1N!9=JBR6$yi7VvYCb4C)bt!xI|56Dk^?!^^b*O^rsImc2jqHKi04Dxjb%>3Jz+^LfOuuWtC+Qsgh@2fGeclD;bq8yKI7| z1s2NIp_cDB;kwvvk}=@inH0t#4$W}OD%qXBZuB5f!{>Qi^+hG&9kyy;opMezQ4GD8omGW+5)7(TCqUX(awcF zBx(of$ilpBKSFE=t9@G}r)U3d#em(H7W2i->D7Lqv!1bw0vE|Su<%i(2tQypPuKwU zwIJXZwvYQQtmheDb~vrhX@pUlh3>unb7gu8nDUW(XTS@UOu3)Uiiqa33nn%a8xmTN z08$Gqgcj=;AVbDLU$_PhU8^q8$fk($s&Tx(Yvww@5(>msUBbs;w`shnS_13TsOk)C z>GF>|z%-!?h(rW|_^$Bz#qo@pb7M5y!{;OSW-w8BcgWf52hXzRjjUjN8w|K)e@Q_K zFPK- z!S*(Le_#gK zcIJT79ITua;Sux%wC+NHNRgwhvPmD*mUPpp&a$jDAFsOI_{_$$=jPqs04++e`lC5l zaLvIv2l+X>8C7MwwGGoO=?;;W?gr^jNu?VpX^@oe25F=vmG16RS|p^T zLy(rP?_9d?d++f+|Gr<0!GOWyy3UyMm`9Wg;R2p(j~MNS%M9U?BWTHyPS$Xmwo8r3 z$2lW%A;Bwg2L!$kpcEoIcS<#IdofPy z4O3K6c-92blVI{wt3LB7H;0Wz^=wsdQin78Dcu-}&Y6ss+?0}bwhL~3-Izn&4^x0g zp9uEUb{TYfJF{Ve{1*>kHfmbB?k*)4P~`{U0bg5z#)M^#Fat2e`Q^~Ip!xHrZc1db zL%lLt;-X>i6mXTv!l}i(E;oP-tJ;k0qIHzcjN;vEv>c;4dAT+*S{1l-sW1GN_WC%a z3sowag)xc@>ofzRYf4za1MQhOutbH6;2qwwAfj2iLb0OZ>KZ^r4n8qG& zAIIJoi{3F{74sv0jKIcHpNS14PAp}Bn-&v$`WW7cqHY}Khhj65C!3}Q!fQlW`I}cc zQP5`1rcfSnAwbq%*#&pDUI9%)sYza*7GMAE(zAzCAds)Sdo{UKemGlcG}h6Ijs9NpkU#ZC^<;%s_Kx$5O{hC@HA%1omlJ(hDY>4ASmQnes+OTTJSs=MD&;qOdrmi z?M@FXKatrxj}VX0ufh$F4R^kg5v3$f0N;$$id()recVs=^8yG!n>Q+2Pu~%6>|7LQ z?%_$WTR%`f$`l6Km8-tU@n=aG61$=!(2zgAsisCp6eh+uOK2ICP5)5jF#Ozzv z6{f0b#1=2~b8uLQ{hdS-KN~7Swm$V5jlU9RlXT*JJylZWLn`Pt!dL?$HA5F@PG)~B z$8sOVt+Gi@$b5NFA9vLZGSy7dbJx`#;AYwWWLjyW^-e`s?oEka)~N=SRnI9Oh_)_s zw$55Ei^MHk=@8m0tL4m;z1{oFvW^AMfiL?Wja)?NSB~5m2pPwFyP;ESVIHq-ndu}W z)2y&mZ2h}>%BFCleLv2VK8h=}SLi_Fqydqt^3@~j2hYb^Df?HDG;T-vJ$DFle3QtA zZgHRSK)J>@Baepl7iX=@cdYAATskpHpK)J;zSO4WZl3?Y{Q^BHtXeGuI^l)3e8lVi z_?p_3#`e;9Yzme77fPhC5gT>ekC2r?^#h}P#ea5;Y?4|8&0?C#Tf(G2TD}2E3#3m# zf%ei*1k?qjCOJ+cn?5?vvH5>h_5N~EdsVrI6fM0G`bkexq$@q%fY<2JvR?gb^We`l z5oBM9p$#i`!U)0H1t6;le^p6W<@>xyc}MfrV<04?#hFviDGJH6qP$^#=yP9)0o_-3 z3s7M5a=x?Kl%l!r_6Af28QPUmXv!B2!klKb0Vv~TK9KFR;(WCDV_V!BTn2ZNV^z8g zhyqCRtP!6%oz)&6mkfDA9k>_d3i6j?S^1leWji-vZK1QOR`idPxXS$9P?CYb-s=K% z94F)ZqcG86gX=w`hu5#vzxk%_Xi&d@ZMPY&LnhaKT25= zrB`unIIVeM_bd$sd==g3jK@tTmttCF+OYU;_h8rx+Jl}meM_7?xXp+;{Rm)QJx= zNW{d`SNo@_29yu_iM3yrmY>vGj%PO*jwMRbr2|vpk%M!vMIT4}=EV8A@rk1_dk^Yg zw>gzXo8tLCt^Uj4voAcq7>z|=k#KZkGzwq;d>H6UR@u9iiObk%H`z7rEv*`*UCKr=H=Q}}CONh*8&RZ*RR)G7XyYsVitvOF^DLHaPuW7w1!%Ev&KuK?)1_P& zEEJP+mqN=}X358AN^aftAKn<)QHTYegZDps|1RoJ9^mag7a8E7Xp91o(w<1~UA`*S zRt!JFj7)Dm{{K{HIB;udR?Flbm_8thymyMft@6Z|CAPB&U^6mLFcm);xqo^Br_4C+ z=U2Y?wQfsS#vc#Yo-Fg~k~>-eQB9R|jX#SIJre=zMVycO&^}g+%-u5E{SROH_;V;>~tfvYX!?*%3%T^;+VxjE~jhp**cme%p_+q%1 z_6=~hVST##1Duj&*TE%_I9Y=PJ(f}>7wztPI8En7%QvZVFT?VbdMs?2wxd83bAWB# z?d=B@@SIf<;*5Gt_LDZM&wf<;9Cjxg%bSYLu<7WWBNsa$-QRqJpDslFBKQXCG2Z;C zaqrH`_@Gvu*T zDJ|%qdUZ zrWd>J;oBeg=+qmg*^8fI6XNT|tn(F{QD8n*X6`0?`wLq{kAFE6z98bAek_S1H(hjH z9fS|0Cprhlo#eQyLHN74@Lu9fC;P28QLHz=#a>LWH)P z{iDA)UT$fk%9L}g(8*_iIp#;kk3hqAE|wV!{~}ypK;4cbRTZ7uCrywwhtWYFeahy6 zsAS1WIJfR4o93+g2em-O4|6DM$_Lc$r({0AzakE6z`zChti2)?1s7;l$AjgeP*+h= zjfih`Zb=D6q&9Xna|Jry*Ln&!o&JvR{S`L-KZY6I`s)ek}HOM1OwuxtV*p-Tvsy=Wn` zeWx?_h)ZeKpoo13tk zDCJ80W~PRNMew+)z^F;|^L7HEzGE1LIsGASkDx$UJiXG4GZ(*#$eOM6Bh7-$`arZ3 zYW-0er%Km`ljh0wnJn&<=4(6=tzSArZC^uaqXUz%guh;HS~yp&Lq)^ud}g1OC^gO|Q;N#aD0=?h*P$<@mbqY~G} zo~1$lXWzHBs0K{WBz|_Z{>pk$SqHAPXIdU1zn59LLte!t&vG_pq1M}`+ud?+%fz3$ z!Gb&*PEvQ2rinoI>8;clN1D(Db&|B{X3UWkObtu@#I-M0z*F)7lzozs)#QAMD=|EbKp zT=a59KhH*8wMs&8x^I%)l+uM?e@v9swNrhTkgpL$#kef>XR)$=a`_6T>95O-hs;-r zgB1ta=d|&; z@8kwaeC#CWWbJmI`m~)7rE&R5R;y8I!Pbue zC&2+9v6eHh)hz)VRC^nz+0nN_=2TO&6!ne&o*3|A{zpW?rqu)@2YPRU(0Lv0yPfcr z!%<6*EXcf%#4mgtSiBc6`avA=cNm!;9-(6Nz$En-clNtIxn9U6D|dO7sCUbX*`I*wWggk{~u-Q!K#luDmMGUo0SKwb+H7#eZxew2=E~gm*gD-vPD*1PO;VON`j>`(((<}G`a^Zsh^+I1vbL_^QBXD4L%K7ZauSG zv5Camwtlz`LU)<$4tCgQIyNo#t*An3TQqgk$juL4EuEzig>fd7F5lTwfBP0zRF=8? z&e8sG70wg-7H&-^nZ=%zqZcq4c2A%x&^HH*a~vn~D+2Pjt^7%SKDregDK-^Pz@;jX zSIs&RbV{%AIZQo%XCG_!IsI)k8zJkoFUA)SSG`UMdoea^{$l8dCmhugiC^Zs1xZF^ z@KB98spuQ+4bVp9)+m_%onuFe)j+gue2V3)gc+OD{OOCZG1Cq&KpQ{iqj9=#x}?&w z&0Bq-hE_)wm#`l`qsf5~e4K@MSZGQ*8~XY&?4J0rc2sw6+zhqR52AeFCe$J>p!zwK zINo515e{EkY(`RtoN3?%+CWEdk7bH=r> zniVGNWn7H-wlbwu*xdLvVVFX@igmue8S||Ev8eY4_LH22tE1!DZ;l3=J%#h*c}M(b zO&!=(2jMX>-9LS!gWb8fql|!+NpYHSFNDY!P$;V3rhL#2Q-@1w1jH1b+;EL$daJ{K z=l4UD{uJ@Pmg+z zF5w<dp%Y>RqK+sG&(>D|D*5ChDtg>+gQdkq@Qf-@t*Xe?37kzK`>B4yG z%lm{zc~&G2UoOV3FcQ&{A2@Z-xaSQn4jRBrmFw|caBg7WT%ziQZ+*h)&cR}J_Y4rP z8hi$!CEIxupYeJEbS=42`ewCA_s? z0|JvxZC}2|wHj zn=d`VF==3_(~qwnehnP4BVfnu`gU~*NAh9l-%(e|?8Lq`{hyol5RZ6NVbF8Brn?1Zg5gkIf? z_hJi9Nz#)hodO{Y3FL;|yNct0wDZ|eDY^~B+{03~0*fcq(WULMmiLkC?v%V+5DR5Z zky83vS<)4+F*EQ;EUT{OT;dB{k$dZnd3#pr=lcEC`uZyVPXJ0eo%Ny?u;wJ#k5^{+ zbl9*zF*P_x*Ngw=*nU*d(-@;}jvardx}{;R$5v7Q0U?OXa*X@1?NnKu#q{_?5*i$H z!KKY$Iaw6uY@l@f9^9%kHjA|{XdrDhFQUHuE?S*dXO*$o(ou0w%70hR-f{mf*n*K&=|`|?O^^<`*C z7)jdYNpDt;`ntFnVo-AD%u%>wca(b;y#zznTl~g$Kr3=TY&03yXxmFbmbeVf#p>5z zUenqgpx*%sN;lDHWvnM#8oBv#z^o~mQken;(WHlZ0T{t;&ePzDX$Qb`N4AB94O<9y zfFm_anrge+IT~l{7@dMF5(}$`DXfx=_FDASXRM1>kOfIP^Y)aVfKqEYy|tg>bJ#SF;YRB zY0L|o_JfJMpjG#7M8YOSSaC|$C*h5ut!g`92<4ojX!VM|UlOLA*r==S;r>SOxl0w9 zkcXq{qXVX|GGQJRN2bhpzLyFu_EF%6B=t`MxB`N&>;PhXu(=7e@=nPO*53yDIp`}w zcGMR&t`?8vJK(;V&tLyk+Xp8c(lybw4S^LD<&}L&WPYAGQ zsg?o>7A(tZJ?*c81uac?LJ&iJ$^1IO3vL+_km;;EYp86RTEPEyzagky-06W$fh)M`cp3Mut!0@X>(8zw87wIk9D?egu~60&$YCi~^HxHR(PIMl0U=^H zSX2A_Wd9zJO^aMxUY2Tha~N`a!{T1>WmmGxbPr*6aFVr91bZ~C53(bc*4YLp=LVB? zXJkRHu(FSRp9^E5AVX7;JpnLN^AexwZnw0OcZEu7K`oHoZ_z^xfjs<85()4>+i5$j zCg=dKeoT)zB^)?CH#P$sk|!yXp-G8mod+d#Pv;Z808C_j3Q(FB$qs4ATwnqEbX1y4dL*c*WMxra(ZaFQ8={L8{>L^O-0Cv_PV*uzzfMar&J$2Jx4d*?;)7+djrh5_b(JkvTohTzM;16Ud33Tz%26DvOeqX0Bt`EexgqXlx{bWz1 zI0yp-PxdMtmx$|BemHIP52T34RkH9%F*IK#B!(HRhg*QSs*fzJ@52TSLgx=GIMW%0 zx)2YwC=#al`UpY}NS^WJx-W1OV?}Ut=jxk{EL=Sv&uyBzM4UWGj)sZGBVxpH$#9P} z=?z8H#DSnXRrX!?sX#2yD&Aw^oM6x&C|0M0o65)ufO_aZHT1Lu%srx$$Cdr&9kDpY zLWu+3rF+lBpcmP!*RIY-U9Ae<1$>c@>^(NMspzFz!NNF!gbm@a^bN}*=j86RZt_+A;}&;yv6lq!~co(Bc$FjG~FAb@qU z;?XxFxWMAz7UZMOt7-<(p%NG1UI& zd`n4{jP7Jct83426<-!EW0%M>qk3+!=a?IOEVRKm)q^PhV$xP<6EycqI2^B(UNA)}yuOz2Ne`sW50oLfKBB`*jRw&UbmCCLumV z;eo?u@PztO(lI59U0a+CivUF($(L8t9;?kCkQZ9+B<{mY8|N%C3Lhax{Ja3cN#^CW zLiRtVIS^1#5kJDfQrJ+Ye4U6U=0by0%kkd4b6na$El!>%Q`lImj=k3D^~s~KmKZ^2 z7N_jcnKD(4XL!(@>7^&rz5>^J$OmSmD4S2b3RmxOm}xJSXmekuHF-`%UV_u9r}&dw zbObl;DAr?q=7u>)qGhnI>XZcCrnY`k4Ru!#;jk9O#S2*Vh4%dP%^s*@<4?935z@xS zGk!FD=Lr}itzxZpVMTF%4s_GVh8{$FF;@rR))++;u4@hF4In_R4xTIvO3 zuY2uK$(8ni*-xoPDOXbQWGH$_A#1__YAol{S<$98lXSkzQMeDp@)Y8 zv&v?ydPbg312|X~&ol#sg4{rtQTM9}1;12V1a0CxXu??#{M7U14j=S!jES zmCO-0bx+KG-)L9foP{b@oaGx<2AsEEL%OuZj|Nr3&wj4QiMp}QI>N@BAx4rW(y6!+ zgsCZbdONJRvD`}LK`hD-lO^WAmj83XtWf(q`n%f-+2-z$-RE`X#CTxj{X%{EAU?J{UFKxWl#NT2gjp53U|jAvioP^oKZ|^+lR0yBK2{t(#`g>b zu5WuwFQ+L8DA$}{ok2#b5c-&8c_qRxKmcazw<|Qa+h>~Z#k{xE~B*K$b{iWNtAMZnCR<7q}V;ca7GY`5mh3sN-6LQV+xU# z$jwG0dGR>7)0Zo2^A4hp;*PK2#LDDp&I@v~@i6Xr7>kq?YA?UKWc>FdSI{okPcF#XO2$%{j{Np)*sdbP(JtvvW)1+ z&9ynH*aIdSg!b#c-7+hapxDuJ=k4%aQ2YLfct=D_c5d~0wFvgvQ5Sp=A@-L&v!92u zS#d%2MzR&RPL=h)^W*InTd*l;BFrg$qN6mLl=TlOHUg^<%sa4hdW)gFd7ZrA`p8>e zn$Db*JumzWqoI&SE7T}Gj9}N1uk?h|1&08ijx`Y{?W2dFB34Hy#o=dv(WboK!=Jb7 z6Ewct?C5qT&+}qs1l`F&!GQJ-PlD6c=Ive6HVwI+pZ(>HFWFqGdx)cefYEb(?*yd8cbKLpq#q4UA zl=Mj~LsGz3?PjW?NzY*-_`N@u-V`fbTXFYJ4zHc=(Yj7RKiMG%E}gQ{!imBA zDC66c=aGY~mw2~*ORl_>5*63KGpXXyVXs1uuF~gDUVHt79JNQwl9es4=$v9i(hzBy z^`m$eFUcwhy=BEKT&l_AUHqG^lTKA+cV8f@Dyd!(5z4{Hxw)GIxanL z?<7ud%=dhj-S@yyZ4a>*)-u>gbuv&~1mhL932XRGU+!@@DMA3dYI=~>BP!LN_Uw4S zti-fp^-2eH>041l1!Id<-zZP&(+P)fhf!0w?on8>P0BE@NUZQ zT|$8~!4pZ7Wsf|@RY7Aq@YKg~!Dz&a_p>V?_kG{Vw%w}-9DeplSP4Z2ND<-0gXc%o zyk|v$Gu>D_k`3NIme`Z$&zk8btfVp~=w_53()+kZJMT9($s{I_HCK9k{5R*pY-3dm z{}-KW4lU1s>2{=F`K=FeiQnCaF-4Jvd(Y0y9NF{!R+vFG&%~bo!^F6Ezo}pErRF)@ z-z01d_H$ogjK31WQ|sueiQ2~8>j#7uI}E{J>|ZMMq;tz46yY+I!A5BwqNsXqj!I~f z9fIIi_Yu=i3>yxM!&atx`l{SKN4xDl?nXM+2{#1ispt=_9=)q@^v1lw{}g(_%5#iz z`PaSXxo6{AJGq7JrR=k4M?86B&f0TAiLyQV*rR)RObBeQL&R8PQ>CMp+|qk3YbEm!!iC8U_Q?3-4R1_$GIWh=PbDEnzHU3&;E7S+g*Pp6Zm-X5t10}c`} zy^LSC)qBVYybAm!_OT7}8H(9>>M54E4X=zToEO50CNkV~zvofmQPI|VU*Y%rE2kFf zZsc?sWJq0P*K!HBsO(H~=rsgkyL?MiS8(Pv1HWZG-sG6MU{y~{Zgl-8g<*@3#BPDo zm+@|~@bir$lN2g1I%!5Z5!87bQlGik<bI}oB6uhR(TX} zS;))ChWs5swuJ0fu@MpLlvQ!?lUo|p^^8#XRT{Q)?j{B|&FY>I*09ta6jU)6#1)R1 z>?QD<8_~mO=5Jx!bo#757Y6!Dagoxlgs=V12FuZ*R5-tepGR)1zj)}i))ZthHxl{w zx;i>O>EZV0kM_6oS4RP^k8tYDiI*7;eDa)^iVI`k-Qx>toh&VAraf7qv?*P|{-G=V zfuizd0VGFR@BnaI;#1F9BNMs`^~Wi%GiB9iI_i=s`Kk*KKcqeMhs0=aY7i#KS7iJzBuAXVf<#(Rq5%x=O`;P_arons9 zHoTr5@}Zm8_CFAezj#d2$!l(~R9Kc^FvM6y>ZzKKU2VqK)zLEFvEACK9!=;-CWLna+oi2wY)Xc|Iv@pd3oRv zbvKYNC#vkk`|-oL*QA^|iC%@*t%$CQL32$}R&A)XxKa6}wIfD$(ux(nbbG>YBX4OK$kG6Ta^A#BSh((&Hg zP`CgN?V_l$iNui>Ge?#EZ=T(1EGd@}*2kgWU%7FHmO56@6d24|ygI@8Ol{WpqLt&F zzR@D#bsrCoQd!lY=*tt4$Du1DnUcLEF+twCej|!y9#th`)LG6VU-;z|O|uo2v=!G* z#zjW5%(DveU4#2WD<3`eh-g*&qPIhUPC5|Q|L9E5L^JPY-`IGI1dQ2y*Bzy1P?Kwb zqi|Q6He0w)7>^n2T1Vdy784V9dzCUlsm4%sHug5Gdrb2{-(xCm8J$vRAM>oWmTl@8 z)sJkiN8g=<=-V*hy6agj$(s;2`X#71GQPpcj~G0Y$sa}W%XBT#f4haP>)JQIG3VcE zWsUrNg*|!iI&>l8*_MuRh}^+d&s~?x=f+zJE+^=qN3-4PC}ft!6#a5M(Glj*F#`%m z*A^vNQrWF-iz>~Zz#O(m|R!>AWZIedFOQ5A|j`qz?e)9tAlAimeScj zsb>|4z@^8-PUB=36h)$s!~FsdZn2H>mDrKy=jVE;X}30rKf#cO zNjQ#UFHF2&R$1R2p+6drFA7Iv;$&`=o zC`U?7M^dY8`C1Pu|LHqSx&t;M*G-BofpMMrkH2dho~(a;_Rkyvnx_mU)=OZC$w_|4 zN6=`hdhv-2&O0$Tc)o6Z%1 z?5e@%2i`EgEESGvuLLKPYI?tqJe*yQbV?Ax2vYh`_L+xsK8^L88h=S3cxej*PL=nkyJLr4mzSEs7TO3FpMHJCxMCr@*+qr0)~ZID)IhlGt^-+ za92u<=;;e%WJI`SoSN^nCfb&@V+;*r3du~{^{hn)UcCu~h`wjaR-*1Z6M3{@@4(|5 zHY=k&s*gl{>_x9j|KI7E|KDy*>OYDG z!^7n2;|6`>%|OQRcj>8Elzv8&Z%P67JT2SR$#-|Rd!h99UqSxABCJTUUKuf>ypBfaq2r+tddix_y-4(7gQ z>o6_n?17%Yzseev6z0Oy94rER1@Z7){g>T-QXVKKO2LXdrt}f#kU6S1pk4lAKgMGJ z3KY9GL5|-sVH0cPvzZ@3casnE#~KuT&2Hlm{}~HFq0V(gur+?g1>)7U96=oU763vq z7{zfvJ&}4s_W$c2tLV6XZHX8@@e4She4eB1sr}67+!4<+Yvk9CUD{-uEOI)+WdYv% z0rzU@IVcKkbqtz9%IM58#eQA@B?O8ykOL(H1~YHB6OJI5L-AQ=xoVDmf@rN@BM}w;&8@f(YD&H)Z zMhx2Q*RlahjsP4ivCA+E4ADU5sCX}I@CUva$T$ZuFb~VAqd<}tZSFzbK zwbJ@8#rz*Fu^OeAzF>(u#})~G6Lt`dMHW_1n{6Oj1*$hUwvsKd=v8NSj)Hkd^sa9y zipmu4`uGxy7h80+80eso!yr>9?{Ws0i=R{@4#tpdf3lh>OBVpV%9EF=P9m|cPvA5W zGC|)qLN5x~_20lWEhrC83X%(iE-WlLq85IlEQ#)_uiw6K1@I#>580au@hwmzJc_*| zHC@;T<}jg7Vtt`cW%AXppd%iD4K`|fp*}XNdV^Mv1dbefcsPw>Wgf{;G^YdffjDyV zx``Td#Zw^B$pFtA$y)xv`{C|F-1XP2%k5WVik5r`H#@>_5j$hT7h!*#{Qt8X|9)`| z5);-Pm_l#DCWSHq&<});3#5MuBWMYG{bmPltNB1ISFX7J#GbbEdY4HHKCoUxMVRM9 z!*U|;vbPJ}?e*~>`Dk7k*=uMa{jE@uuohUa5O*{J{s#PDVj>C_JCrEtOk@k39Y}sD z>UU94n$l_A1HV;4dT|3i+iGdYOgm&INrcRe98U8{++g%4{Oo#ebdXJ4lj&&%DEAwDA@MQtm|JE)uA9U1=Tiu0Gl3>sw>dz ze9p3!keu5PQZzD*84w1XtD50#qH^w<)?>Nah{WYvnZuV+N1@>n%~%sR+=S1(l>f>- z|FwL8-Ula?6wwYQ!Ksxr<-`an`p)uPRudGpVD`ozKx#cM`p|Y$O-{SEDh%Wi%5nrE zbb||ajae>y5fj*l`oN6ET~y%ELFL)cr!K&kiMaXTULe>>;%IV4*M>zPdNK6$CLL?! zQADnCY~sAB6-*^Ilf1sY$ijDbZL7f)LDKAiA zW+kWnhCvlCLk8i9FhQVNFV%ErbdFj9Ss!j(KrjJPA;{nU;vjUuLTv&>^+kql2f%!` z0IS%&f`OW;Is!7$7xKZjc9SMcXbM$rhHieqhHwA|L`J3o>3D?SGiM`aq-v=?b=Kvk z3_gbg1lwS`9#T5ZAgC~}^eqsSy7~0J$H|uD%}3DB9U_9fO^QmeQrSgr1=(E9HcHFY zUcU{BS=)b4T7U|UU6#}}srDc`C}*mCVe;=5oU~!~Lx27YJ!xoW|I)G)$1oW$a-($uc-1UDZWRncl|B;DTIw-tg9 zXu)<6gCvBPpM&iWx>~3IYyON{!6}?)fcji6g~?1e{o!xnk#8s24hsZ}NwN`Cf@C$uNxM z-G+ulmoXFae=AyUnI-8QWYc3+qD?UVxX3{}l%4`F#we zm#uR<@8H<9GUy&TXCGg305$`+|@QaaQ>xk=wNTpW}iBdd))C`<*UxQb=q zJFEqm3m{3N^lj61OCNeiW6vdlj7Jc;BRug+o9AiZyUQDKQ~0Ih!QI1DnbJugKK+kL^-;7)-S2GMu0{8ZLhZKHpDm&-|(G_v36B z)tb~#@Jv?0;(a~eU}pnPlk%0>PcKXPUs{t%Q(*tyyi$zQ3PZ6lhBU!Y?c1E)k)GdT zSued=QX=D$p}^n=*Fo7oT&4BYGDGD%lrpmO`I`UxbSV%Z+z&wJ+`J5gIAjo5Y>0A> zi14AD$qUH3R+&^!%2QL59s=0hgulNk=0oXEV#VCmf2~*6ZGHnN9D=1z%U+p2zEEcE zhB4UDWROP-^){9u!BqGZNMpX-Rf)Y&M@WzoONvB!X{AtusjIaLscL}qzf_99J!?A~ z9K8oK62qC*7cK2I7&1(8F@Gkl1FYKp(Tqf8RG0I0|L0>*k;AdOgkh4~%fviRriG03 zzv})l-gsc_7sE3HK?s04-FL&O%Ku&6xhB+Ix^XASo&3QeRO5u$c(Fjcsgs@mhDx+- zQuJyA_X2e`BJc|s3Xnrk1c6%`FK`X|z8|*t%}iQs?G-O=X>F+EG~ah)cxcQ?OI{T% zH^DRDx|I3WcYwrwi3Kr3p;1f0sNUB3FWBVFlUiBSf`f+RzMeWR`(7Ylgu`Xf&xYXk zGDwGgovzk#%(m9?ppw?yDCIJJO5#TG?w*O0^eZCDWw(umw)k}q8GS(>HAV4KutgbS(N7J zB`(8~L)ndI0c%2xsDyw-E!_Gab4C$_DiLiEaj?jyXIM5VEq4h{pvPr1)y9@loWm>K zK}+Loac?KVd~wRQ<_m3i1CZN49-ly#zo7n0&7tP9HXWs``YJ<0>>Cr~b#Y+ul~Svv zJrYBiWH|EAJG2yifn#R=h8dsd6BI126 zMRCDDG?W)u0R$|%M7~h5kySE(0{}GzP5U6HpzN;v7P+M9(&%jG#L(zfqK46)x5%~tWO zp^|-{u7(|@F^iz3FZ<`afyR!$GOz29BJ;RT2Fbz(aD2=B&)FJ)6xP)ff=cT0BG})y z6X<}WolMnM^)#yDXjZ7Am=oS<;r{bK2k1(QEvC5b&9Ic)XcL)YAL{j+VT*#I3h*vZYy zKoXf~g5ETxu9yzI#1PR`>yswZ|5Kq(sA9)AH3EW#JG<0ktS?YD1)SYNHR0c@&6WBIXUOwcg9VyCUnyKLZ_tpQ|0TWeNU2sQ0;>QKG zeU13FYRfogQYli?b&tT7mK!? zAUePTbY|A9azGHNbHOaF3aU!M_W)4k_YHbq%laQ~nlskC^`EDPBlYK~#Vu>GeL0;3 z{pg-s!|RybT+43-_xv-s4SJYs=Ta;_irVck4-wdWon&rNa;g1{GpdwW6qP|6HBZT6 z%9Ctsiy3qH4u7B~M{X`vHc{6~z4OjO9hD$03IFn**RtWtZVQd;VeoKt(P1DhT@!}i zp~wB5&((hMPYb2W_UAXP*>xO{2b^?S-2eBcB8L_ETm`x9e7F%k1^l55P^-_2+J(bH z)@y)l)*dKW7wS&E{;dz`;ruB47k;7he($=AeYQ*QZhEru=wA7bsyxJqr|(_J zYq4pj%)|6+=^F~%S>cBhjY1)ovFjrruB&Xlk#UE6BN=#9zq@{K*R^ho_Ce${i~ZSg z{>n%HnL_;Y1*S)oC;^PE)PV-Q5|&_@Nf2@3IKB)*ExzgCCsl~FoHqF>r-7CN5 zYKNDSX8r~n8GxDL>ZBr@0pVR?BF}NM(BX}|#qBhqMW96xVPpHba_#k%7LP~X3N(5G z%E0l3((Gsnc+zUXGzS6Tc2W#=wyAcbIoY1W%xBxrIX_*U4Udl&EL0B5RZ8{1?IG2J zLfAeN^dOeHiA`Tu-shdAaHEq6efMW$FmiI}j|hBd{b~d&Ep{L;&$&>oav6H}FJtwwJ90 zU`x;%hLbO_KX?P}772%|^42Hg2Y_+naP#Udpp2ngYd<6Sb1+28{5cq+DoO$oZF-V- z`Z6lHQT@^rIes;0{V>)1?(z`*J2Z`bd_e07ka3de7At4%nudcgWwQy(;FpDyYb<){ zt_Pk2(ISpor+c$T1urk|d>S+G-<~@=(R@=*x`I(`2ejgWdaB22lTXDPn=Qbt_O2kI zQS|P2q5#lKWV#tN3~i0)I6&v5po2Q&f;P23!O7h4>wN`eL)sX`@^-f?Z7ti{s_J+A zQ%VD1O=cyEt%Ds?^tKA>4l%ig*rP9%ju8dFM#y6~UpH?seg5B5h62P(sWT&79cM*g zVc9%DdwOr+BU0qB+;IS;@g&6?E`TLO??)YM!a|0!u1B<5|6O)J@lRcs{P*l+nsckR zdzp1A_z?ERVjfk+;&)kAh#B*K>f8jG2fORP;@V$mU!&rF%%F@IOZqIpk>Z0N9<~EP;KDgz6U@+=L{{a0ARooI6GDeyYUs1-@0k z#89XY`&*j46`msdgVd_({4Gst$_Zz6rxAlhMXvniLl$RP8oU*2=hIUO_&z*l3Q^1l zr*)HI-e?<0sF6B%jG#$@Oy1p5R3o|H`d;4@y4CC4oIT&8ez2!qb~4j_sgdpVxy*U@ z3Lj`2p0j`E!FP`WT46(7(J0}j)ld>>w9H22`uT3o!)>tusDP-_O zF&I#X>!ENM1+8)6i_B&K#g78Q9YLGrDlHxpd{VJ7?I)WFPD6xp-G2(I5-d3SVtXiQ z)*ZMzqyY+NmnX8=PvQ>vieBJWApycA7Se!k;o|MSgCEx!UgJjzcFl5P6K_=Nw%4Rz zL;tMX670WEP^KlhmgHp|2mf86CFirIwYtRFM-k|Rt+ei!5*B$2 z+$^@t%3Hjn$tYYsTZqmXjdoXzGEdVqiH65U`;hko^pd&31&Q%V{S+hYl->s*s=_Z0 z4<*48>;!kVu@lp1y;)FLs6?jPk+_Sbmc#2h>0ot{F2Eu`!MaWQsY{TySESxPzPF`r zEsGz36wn3qmDXdx>lcdB2^5iM<9cAM(*6cslW#cS!AWiig)8wP&%%P(AR!~G3IiHWGE;e+^hDV!;)PFF$i%i@Jhp% zz0>!!mB&DJ>NR?K(}PL3N&Dd?nx8_pgd@ z9|nW6)4Jb~QA^fW&!3F|y&|~G2GG7$0=q?kIKBrLVg@k1$^b4;O!hAMk?347N5 zo^BK00Y$Rg_@zO_{YigGw7@spB@Bz>WX&--Hd>SG-#-sFVHJfJvG65P30b)TTbIj< zYrVz^3QRp7wOx;QFoyy?eGCaan8%w{v`-t1CONu2;`7s`N1k0rRF%FMmOcvnQah=@ zgx^1<Uobe$aJPAvz1n zQzQy*qZp*AK;^(`W`o9_m!(9kUMFOZueK-(E3f&o)nP*3q*Jr=r=yOXsYf6`G83o9 zp>!(8h21Cya)kq@Esf0jM~jX=sP<6tlHRujBFlt7R#;W{E(R`NZ?8_#J&92{#G_)| zUn;tMOgTK&pOqMgzL9-uvBiA=Vk|*z2o6Z!k*(jsTo?l|w9JK0^Co|S>|erfq! z3Mnk#hg*@m-x@Apg=d73d40z6OM@bi2I2U4rB?(RK&l-|=N7CZ+^oi$~rM0ra;%4T~obkI{g4d14U)SACmAV5YALnZ(H8{ zOt_xyyiZn}1w}(fP4vbK-~^%$+?$++UVlX}FALFnogc%iFef&|Kr+r$4~T$Bn^4Iv z#WJyF=ISKzHv{*j&1-w&gKp0~Wy)!wjGTg6cjS6k3-X34Mpj>XaL6#>z4@-YUzTJZ zE33H-?9y{pfVkt0XIT}4xq(Ol6uC)qg~!iq46L65!=u!i#|H0QrD?|QZKdWHjVq$R ze>8hBBQ=SHMil>gsm1lERw6t1K~qmyjxG-Wdy~ruOXmpSM{B+?DpRG{G%{lfXZ@6| zAMyj-7I?GKR}80V<@lM!fZ|j2`Ks@>ip_(ebTp?%_m+W)U72b4?qXZ?LBKpDiZusl zBHpY5LYb3GxDcR*s>f&B4R)v^T{RnlisOAsedfhmT}e-jwBDJ`^j5ZXLXi~UJmfhv zASfTc*aRd?0u;;dliVwNHTrn6w5%#~dz;1ClM-G~3%RFl;(pJ)S*-WnB80 z(B@ldjwqRRxGMlY=y&`1Jd^&&SrF%><+Ts+9xR-1bjYcBoCS+iMm;l0oiFrA^D^rh zcSg-(qHmNNZIf{(boNYev41D}bX5m))DX=@D(IA!blGuH}KfQeA#Pf8X$hd%Evkx9M^1R?)Kx{xv+8wC}q9A6!UQu2j>d=TXWFtU=v zN3=7sAoV`#7T8O51E!I{mL-Gz%0ur0_9IBVD(DHs`6$@{A?Sh2acp-M@|&NEzT&X` zHofs?O>ZmYckTCmpeNCJ18U#dPNe-_p$tKH`>(E7h~?56Lr4e8xKlDQwR3l{-iRj6?-FG9zcW)tY@51DBT;56`v`< z&QP(|$RL;{V44tL#IQ-Fy4hr=kdw6k`)w&7?_CS) zefQy%L|~3Sr4R$-OE=j9k$?X^CNaF%;?dk1L3iXwqVjUKDI`ht=82?NhL!Rozd=kP zC<)NU0ra4A8_232gC$0cVYL`eJ1J8VC2@^@ZpX^p?#(%J8ToeUSBrP6y3>Zzq)QyqeJS;e8rGoe$qjCFh5a`)8O-V(fH8Pd59TM@?pjBA z?jN+io%+0AEo@@zQFKefE|yBy_veB4FP%P}=~^;1^%7J*Pzx#>AJTL{bvN4V~1J$`wiqGD}&0=cHh9FquWY;#(jb8QlM0#I?@|2Vf z%GRW;H;xt*PjJ-gztn5U#$SIrF52$n%4CNm9#QL71u7OlOQe)-P=}VYJ*m02_7TKn;%1Z4Vd3JyI``lzLN^%?RfXOB_AZEhn^UsCk*;`JJ!LN&n zSe{j_C^Ys2_LUxWv59gkRfU7!AGOTAxPM5CclWg52%RZE{|ODy(^GN z>4=O;x;dWw67Xw0PJ0==0viQ0Q@otKduL2Q4QjIqIg;o({R@5L0sz|oCJNPNEyI4z^5`N7CTCVW7 ztsXoO_dqcwH>HqxTXx@NA>bO4Pz#INP+s6uCdq2#*< z72YM4)v9WhCY&&BvtCA2gupizU5dpy`)l%+OK*^h<_XOiRy)cO2XZOrl8yx`9^-W# zVS}QHsotQuUy+ZzK@}wDao! ziD22i7B>#;v}CKq9aIU_H|) zILt`9W>;0Rcwk_ejq<)eXgqQcwr>MI=$D&~wK&@5nO$*@>`gy#p{We$#JU5p(1H61 zSMha>l%-6NK+?5X2eKd|#@m5mcO-PzrPz_jLxx`gW&=N;;eT32K+Ix7(2ZXB6btQC zT$W$82P#N>^2*`XFBQwOZ={mKZ>n&lEs3iWG;wT_AU7LS_b!wTM{kig5&2F4cVASPu= zvRM5DG%UaC7Y|$VSZ>6VRK8-T`=)RHT&Ib8tPO`d<13}@nsxTpxK{kAP09nai#gN& zWiqJm)fC5a<5W@U^*yVr30Ke(!a#IAN@L^%5>w1lk70~A%Y4ae9#`qcS_v4S`>?am z)mZeuWhc_iFk6z*m(Wzi=UM5OK_6kNC5{^=6B_H1+^|!D8k5HiA1)~7YaM$2j8dtj z&>i|&$2U>yf`_ivg;hCOQ`z4}1cPCZs29LCx=*GD(8qjM%;J#^kQ-1}_9|udZ)Eht z4LysUB!5ou+K0XvzPvp;{xef5C5%}$3YNhM(%*u~>`r9QD;yk@JiFX+d~c@? zQcF~6T)_w3os<>D@Bctnp@@}IliurQPY3E2S#yaif~+6FfnJJDJTLMTgJgcrcKo+p z0dS@!>#^MOw=1qa9BcM!sPztp+CIkq*cp zZEh_XLMxw2^l{dnmo^~Uvl83IeUEm&n3M3L;mc;A>OmZyH?E8HkM@e#VXV`yhF=ih zKfrg)3MiH^&90*DMNN+~GIj$|BxH$&0+ePzvAnv(0RStRp|n{wx^iFZEFv^?5YkfFlbnPOVMj@Nv8XAA3eNaF>^{oeq6-GnjTFf?hM@U0*PJ9qj zk5kPW?qa)(cRFFn&_*xe(q~m!5a}QGX&zyb!TsZQF8bQZ9nS%et9?#lrHAdRF5VV` zam+T|c4rHEeVRw)vDV8QQ|Y0)`YtVTl6&r_G1|Fy z;@<^*2$aWcVn&s^B_eBKcyAECB9l3+E~9ws!8~mAQRz(cd^!VgvhI6d@Y#PsvA(Lf z7vzh?5bQEIzyG&6B_#k2x)0Lgzk$NspaCeIM8joAVj3{TTBQ&Ch+*(fpPJHjlQTf% zXmB}=6f=L3afjOH;%_#Ha!s^*7O$!oRjTUDTmq13pAvG6WA6bCh>AOnkM#@}`cA(Y zffI>lb@qeyr}{3VYBQt=E8Q%!YCOgG7KhF`ePPoNgZkgYsz{1|;t%k<;-S+#Poc*m zCRBjj)VQ(AL&GAxI`zHu+fe)W-SOD^Ali1)+a0JoEN5PzbXR!pWaf2AAJS7&g>);Z z|D>h{CKD&3OfL1@qSlK5o#&)DIuUva2KJ9*+x>T$Q2Lrj6-Utzu%onyeuDc3!$=`4J4iD-zn+MUe?&@Op+8R9v3=(lL(@-mNaaLs-6pB|G5n9^SZNH6gX}LXbJgsF`*hb65n*|FT zzV*aO*a)4Tu|(!_VSVexSaEoI+P@|~8d4Bgz5;JI@t^(tCuJ~**3Ny%6M0CFc?h<3 zTj^((@7?hRFw4A``E67$(vhiSPGI(BT0#v5;FB=J!<%ydILy?rDrRTp4fDanuvRu< z^}eip=ZYM@WNrOsA|)s`s3k0&KsOi~LSXD3fy(jYfvf$6*p1VR=K#*pq06PIj-0Lx zZ@{e@SsAquFbf0kuT39(jyos24@$%V%T&p_6)L01H{nsgD`LeX9i719?NAS(mzWm` z1>9T|8k#Ss21#WRDbZw$nr*y4wIC75EHSP%VxM&QrFveB6^02xs|~mXRe?z(=(or2 zPM5}74k(Fs9+4K&8oFLWikhL--LIEi6-6pPI8LAYR`(PsL1ysQ1A;FNAi30tQS z`r-8*(w>^Z+vey6x#8XBX4onAKCIFSCx8whvbh~ zKcg2E0Wn3MZJVQJ`3A)Dq3dBQPxYS&tYxllp)IrK|6EPB-iY5MCEEFc*TFG>j71wZ zt~V}O@FMWW$9aYm;^fsDP z{Kf&rLZ90_BQct#dZA^icNC70hmeK6*X-R2D7PZM7)!YJhJ8>*UAYt*BD;4%y3;Gk z1f0!%zzRo}MljTEWiDtmtF9B7kRL-4ZBAj_ehKoKXe$^FvZ=0BdDom$0sBO}zpA(- z-EyQCt_Z?O;R(bY>H~6^WW(=puNw5@pjADHIfYS0+U9M(a~w1Fg|2BQgK=lw8^xla z6V2MQJ|7V^@zxVF0cHfru%J#OtLMXjYX3~H130K2(h6J4hkZE@Vneu$nGSS}Wn5*4 zh;&qP98$r79nIT?=sHDJ^o%?6$NPU81m=id8E-&<+V+D2{=Tmseu8QxRl4O3r4Tz} zK(0uyYR4AqX)oPvJvySSY(C#lJih(b!7-pD5HIYL5*WxoVWl*H`KdxhRM_tU&3nPpXaQ)UQ%w>kp3Nxf60rOa0%&T_TPC9 z%(-ew6bA4yXYcb~=bZAc`eRKEJlPIvmlgdQpg%+#hX7`-MURW zQ)e~O@O*;>>v`^wdml7Jm06wpFlh(G0rZ~9v#(9FPK9D1%5gLAStvx1K4n(se!|Yi zzKcd z_1DQ6)8|w0^rVn~oG%4#ma{~-!;lWLQ{y!g$}M`tunxBC#nLlLEedG~!~maL2EIS* zGh-pP;l3K#cUQ5j#G@!R`G1deQaEMA2uDaRk>kz&T>-b`cy|wO2+;Mc6#+?jdEMMb zH2cuass!Cmml;So^*z z0Hj08Xt6|>Xjj^cUn^Z*B;00Fx*uPy1~EN0?OFPWpXhl52@Tg&NIUUPxjp_i-dG@) zd*u2cfG%J>@9wq-WOmyzn92k7WDMM|!wxdsI>;L337TmQQ@DomPGmQ3qI+i%A%fhB zWqz7DnUkI+*42;A|&P7 z?7@i8-nIt65+6ol#J!1-(Z@ON%|nRA)PL*k20X3p;hC<>pmD3F-Jf{>EcsaP3AYz` zrpZigLLxBHnOkbO(U0NwL8__fCVV71M^THD9z$1`{G%uoYV^m3_AN;9vXJsgMDw0V zFO4X(Ng`t^V6Yb0Bh(p)eyuV4=;`?NH~It5u1@3Y&TyVA7XiO#kc=HOm7 z*Z0Xx!gD1Z?#QI2q#l@7Ej);~3-U{fto@4Jo~K?jPlb?>*0%N^Y!(34<*I!DkZ$BY z7VRm--hYI~C?k!yu-VUw0Z*>*NS{Jxv!|DsautSnI73A0)b!Q*K}L0valKFs3!}*p z_S3lN)vqpMj(TR8b-@WOqCUq(;0W^_MaLk5KK~9nhH9F(psOZu#DMcFd>6#{y4}#$ zLc=n{79L+ELaB;Kijtu=wdxlfFjmw;l3J#39C015PT^y%@$U>go4#0}d&c+R zu?+(Pz_bTyG37iPhJRS3@(`8S@R|wc@(D0CZ>K#L!TxBpjZCmw3f+@smreFGD`h@& zxm~5;eGgULsUMFE)wKLfw^vvhd&YYJ_B~Y$`a)G&9&t>g^_rknye!oCCh3a^4>*oC z8%DJ*jv>|9WL}HDDB{+D30hLfl1*m@yiht6eHVxdG?66VKrFdci9i@)HC+zk4Yza` zsN(Y&a4maJ%BDkl((+`@?QjL0VcBS63YLS|buk9wVP-J(%)N2(ZVS@HfuEk>ne@}$pA~BN|YvD{-46a>+wvl2GKfw5PbgBA@UbUX_ zQAgrH4>1&airTaKP1jO&$@QdGTd7LXlsk%Ai1X5$EXSmcfRT1E_6eYLF5|evM&lpyQx{ zOqq=FCzc;dGthp@=npib9y~p^iwvi@;bKSeS1y+f$h-WLYBN)SJ*2V9SG+pm6`qA| z7Fx=*iJq2C6efEbeTHsMqwo}B6^Tik zb-nO#XV+3aeQ>2(rU1mlqCpyv6?~oE@_gM?Tm(kBH%;&`eA6d1j)Ddbu3s^mK3ryp z6t2EVpkQC@bKp}He1AzD=SHXxEP8vdus%uoDVkfE?Dd*k)3pleW{ymh8b;Buy6 zr`UjEH$E60KB@K*Oj}hfc=Bu;?J8{TFu=uUwxsplQZ{p(pk_c&rt*gsZFF3pdW(#B zCT>-L`S)t^94jbSnpj$t5OZIwHpAzFnct6)lKuMp{#-9XCA)u`b8fY(`8)>)V^T&0 z8`NHW%RCuoh&u|AjLmVmMveLNQ{&lpYZ^<}*NYv!-LWqP zLoZ*#yrh@>EIi~*Wa5~Z75XLG0%y?%L%MFgXCcOet532$rcQyHu}?^hxi?z*vRGtY4CQ8wUk^XEBr7n;)zx z0=2&l*|cRfL9N=>^kn88$uo{OWD^$5#x9_a9ba%0{w_HXGG(gczmU9%Hy|*5Wh`Ni zgXMZkpxi(enWeEsPxvY%9XIx$kuUrl$#p@2>7S>8r6Y4YWtWpBySO6P#y^!Po!qLT zdVi#{zgv4Hsm^n)?}tX%M@C90;R zhUG&vo5VUZ5aHv2LP(lTFh;|!DNjt6&61V{9*J_^9hw9AFqbpdMb#9-W!Mw>q;bMb z>~va#8LzLhUaT_K+X#eI>GUBlik~67?nkSqC+g?Yn{jc)cNU78e)SPC<#Ja#8uctt zk$Bj`r2OfEUc%3tiQO-!!PeLEqS$GUNtD9x1bKPV1hR+3o-lG!UzmQuk!cDteV>s_!a??NCeij`K=6F)V{5nU9s$J$ew z169Pmnq;9w^2A+HM#?x^Pb4~=qD7U$Ub%WL>c3AW3Q>r5y^3r1K*O!#%b5 zeP)og_eW8iVh6tAE`cl_$~j%~wm56=lRh>S=meG=2!C^Hi00jvMxns}Dz@?>Ac~Sb zZ%y60IDtYG`Tg%Zj6&uStW&D5s0$^_OCNRhxWD|uo-u3XMR$BtYk?|FWIeowlys#Z z*5D1RsqOGAz3binrj5*Ae#!-S8OY7#0dx8V_IY@I)PH_xkde%-X)+`8nF z*At$Et~apRTkG8UWU06UNe~JoL6H2Z5Y!ld<_#o%x5Yufo8l+GLGozJ1+|B`@V;LU zg#C@sg5S)w4c}#TCsO8^-fDBfoCP(4Jv@=!VDmxiGo(H4ksSYmpLyk z?s{AK($4eP@#vEu91#MO)3?fSoIlY7)Y&I2`6FjRLOZ7neiBR_vamOsbSD>n3SQX6ua~96vaQR0FMd*uC7jhA; zcOcA9_|$rdMVcMXf+2In=Sjm$=Fmcv0dY>$qwvuX7G15o_3R3XpEzil$3(9O*kYf7 z6qVAbZzt3=E^M64w&;~mdhj6=)K0_r$JYY1t|CP>SEj>bVEgX{a9uTQ?!w&z1z1K3 zpV0a44|5RXdlogQtm@Km`_2^btH}FjrV{Eamje^d?j$jfO+^fI3j_|CGEK0D!DhDRTzWf z@vLnHwZ@D5)%T>1R0ni4s^8_A`!8CEj!olN;XQlx6=SFiE)^XVvop06X9c6L#mko+ zY>lY~fJ>xZWW<77_B!G^JE(hU$5<#V9r1twHciWJc6R6-XgVC$96))W6M5>Y8mDwK zjK^#OQ88UvXfL@o>cR{(J-I+w0WS|%2g7qlPS(CgX)w6=eF;f%sUe81rx=8}CvCf> zCvv#9?k?;$?;l=!-}pzy$C!81$>1zLS_%wuM=fXiVEg=u z4xiuJksHJ2zl=Sl|HqPTVIiuaxsOV*P z;2e1~;FEHP!QlsZtIM?B0hh6dSVo!id=ABv?8l5N2{2+VZ9dEOj zlFEKsZdNW^^X9aai5+)B*u?MCs{xK#6>`~&?IV_B37IFHUI~y%9`<18!TFtMDuG{M zLO6F!9{4_zQ}5JE|LiyiUK_Djn04k6lj+#y2Hbz05Tf#1PVtIDU!WQMx*hUn!O;y8 zWf4{x`FYMbIOo@TN_r(6dO-$Dd-uyR?cwOp=if6(+n&cZtb9=!pYO1M`+pWdlu0Q; zM9bZjBAf9O3b!Ge9v)jwNqd&8I>yOlM}3u~G>_Wsx>%JjFWE{5oio2cBYjjQ{E^w{~EU=*+^ZIo1onLHc7}htOZx;a>zP7#D6*#*hwR`#mBIPDdGUqyw&AR{e$<@FrAo^D$eR za`KKF*?(U3dm|CaRx=;VQjvr+n~W(`k)BBq|ycWP`~Uen+GRSwJM{~a^X!GX7bl2kVzdT{%hl(eKW zGne(xJHec$11bv2`RNaWk9AN!1rbq;;p`c(IVY}NCsR>6NYsUt7CLz?$>{zdC^3@v zR2pw@bQKJB)8&z%lk9l3hr~Waw~<1#G63$eA@P)#d#Xfg3>%V>{`9j$|W;uuCV2*m!3rFamqP> z;*c35$l+v!wbb<@UM<>x^q7CDuUV8AJV)VC$PuO>P<NLU+SYTM!v6qdrBs?_vq9W2%ZB!0oyk4u_e{gKJ8XD!#VQ8QZt;aFUlCXF`^mE zh~I>Ki`NS$s3i7`PZV!o4f?g@4!Oj~+U5R!##2=bOy2WWrmD$l0|70RYQ@^kEz&On zsq4uPHH3j{zFg&!LIU~jJZa7E?>ri5-tn*T_ntI~Bp^q}armt$2uz%1coTVWMUJd) zBcf}ujk>+BnH*JY8B7!Oi|1wmorqkiQ;c8e3>NJc>9C~-qj0E*l=$eQZSHfjOSgC) zrC&CB191eNoUr|nO{@LwE30;O;XzWhJWc!h2JI1dmIHT{fwx9rVJxFU^vV?3O4A+% zb=cnHcwfu>m#ynh7AAf)yzi&x_Jj%ZZAVdBgS#8cQf&p!0{^f>6hh%{(Aj2P&7*-v zdgx`edkaapW%?$c856;wu^@ux8UVVoHDk8F+3l~jeD@~Fj<{C+c4kTc=qkgEF%cyqGiSI8`L z`k?$cY>7w^i>c#{g5-gx=#Tmg=Uw&jX^iEb=wk4REm4nJA;!1Z3=`@L8ox^!eCXcA z=0&aQtfxJzyLB<@6Uuhk*s}f1ao<*8uy^%4E83QXbQS)b`V={{Ni3)@?`5s*Rz9)l z>q_53SJP%doXs#3TB78_6T|nGI>9AB)4<9!Z#D0cW0uW!fTg>LB(S-5DK_S^VG23) z>Ao@B- zlF7FWpqTq|5BiOH_O=#?r&F<_>}l?4q+AH@UDpGCu3p%D&eIolp*26VGjq}tB}#wo z+dTTWkIn#?G0QVQVS*+x(BPMF5{j{ZvU}zg)QxLjYzRdQS!f_Cut-%TZXR2eNQAM+JB7-&0YTd3!_XP%CRfSH&gr{- z&RiWAz87+4UA4Rrp6~(p(!jn(%j#3Ks3+^xN4qA?3il851x9ln){9NF8tM?}w>Fhb zPUIqL-5W3uT!e&g2?~DoBOP(rHQ!hkon@{NBd<~_SBzB z2;A%MFOK`i>H@z?6lYK!9A+jHrmvET0{51;KvgF+>-bLUbcCpw_wlEtCQ+*rTbT{1 z?ykYx+W{UQiqijg5`u{`{qtg)ID0Slpa+h$rDeFV-g)@KI|foTgs!;DQ9su&`PO&X zsceEJ?{a0PAJDJ=YjD|0+t|4Tar30~WF`g~hCfsL(F(H$$ zww|#q<_o_2cVrgv6%2!D+l#?vK@d$APF=5MFU%$I z8{qVG5Ez4qI?g40lG?a_G>s^C03%B921BP~;Ax=woNSba7amEJ&>zih0ud^Gutkz9 zI5~p=X(ry9SbCBL*w!9_bIj80E{oO zhpwX2-i&~@UrxqYqN67ec*r^h-bQt9Xz5%5q>fRk)i=vTafD#HBCMvB0N>QHli$F( z$+wg5>!gvN6)B^11`a6oPs^VMVqIKo`X7A+xjgB5+&jkA9RU~oFg1wlexLlil3hVO zcy3sQ2{3HMV$G!+*l%eORv2qfap1gkNT1>4UU?IsV;FI{qRwwhxiVAfJ~3lGWtHe7!T7^PJaWK*0oeE;wJ&If^pVwzKPEm~c|}_c-5A{XF$SxJR*lr8LkGfH1*)y zBSrO;s&6Hw!+C(Uw7aYNayv9|nTu%d`;l?fuxyU=*6SZdk)$<)PkD7G8)z1|j4lWA zIbkzCaa==<6>VloY7I+{&U{fiQMChiY|}Dw7dk2q zzEfu@0Vfw3=i5P#h@j;|kJmj7?f==d2^3c!G7(%6FKsqAs1_u^_}Q!e>mtl~LIigO ze~V6GH|NuD>N6G7wcAEcdg-qlHUC{*ry=ZM6;fl_1M`C1AZJ48VaT-=96 z65}I#%Tm15+5ki($)(ooYt6IbA55EvDu!Nr3xg7;Q|})#MNrWnC|580&jBBUBtWwP z%y){ct(ZqrWVreOb|Zf%GA6J7O|;do(+NrE3wfL(3`(CLG@&Xp+#b8r{RZT?2=Mkj zz>?bXy7y9eyB%4mZ0(xm^POBHS$YO&7Tag2bgoUn2TGO)uK5(7Bq?C{RLa(}zLD^6 zGitu+DDXRp{%TVke?$NM910~GuEX+q|I?zG5iyi{?$2U?wN9cVuY`VJ?P682kAuCg z$R&CGy0l`he2ySy5KtsNYd(se#dg1$P{?xC%X7BIBq0}x5@dQ~YwgcZe`4I!XIfc7 zA0R0nwP`rW-VBSCJAyUq-+;&YJG5qWSiWy3wF7d=UEV#+86*6_zJF~0QJ$SD&6`;M zubqJxH69Su z7Ro09XbPJxqh)OW`cN`m-q8_2ckvV}#4C@~gCj%Rmpn+#m&CX9*LcqYbqsMGLz4aK zt=gNW1zlCGBsO8GH?s`h_2vo^OH=@-Re`)v`3MDz{xi<0BXC+jt$&>E-gXNhG3P&@ zaewLKB>x~8k=i*J!t1&&$5b`ZtY4bReqb{nd6>)Pv+t;N%Ii))^ow>Majf&F8j>ggN#i(T9N9oruO|WF74g>F#Uq zNwV!dZ7iXOk!qq5`AlbX%gD0Ir~t!x$@cYtmgm^tMaVM)PeAu3IS@Ew!69 zo0FQsnu%S}Ek~Af_@~oRm3FJNY zu!G1>^ZflUNqF$=J<$cIwM`o_&C6wsNTESPk-skNxp=hi&dikN&CXs+lllf`>euHP zY^?9SF{{>R-@d=_r>5_F3vDj#PjMG*vwd0rh5*+>^UE)FGC|>H?0$>lt$z#nfhZ}E z>N5dGYTZEuc}{;->~BR%@4jDM_jmEfZa=bQA)};srcC(8!oJo&RJX7*mtVNoFIztP z{k`>M(mpmuYK%6hy$xu}qkpH{!Eo>cG|?I;3~0pm0^eNsvuKk*^e5OB!20W{>j5rS zVeH7f20My3c7OV^EVlRy;Db*>oSbI;XPtCJd>Eg6$?&Dq-xr+gvlb5=GM42PX~$Ap zQ$DwN8lRrzjw+*2Su&m<+P}m0lE}FJto%pwn=)1~!-C;p<*P5Q@8$v`4Kr(EGj)#F z*_s3&%Jpl_sU_46WAff?Dh#&+#hrU@me;o}VtE2DYObHQ9J|%)kLc>!e4R7EdwZ*D z^snRhfit0l8w_SfA4Ab#VbYIGlyI^Hem&yb&bl3w*?Zmf^F^;nWx$#MD}DpfIyh9i z!+cvA_0J#N+HS=5?ZB*_3nq1zwjPqqip|?7L85iS)KaAZAJHi@*`vJQ1bW8D_g~ww zj+o0iePmI?a1DQaI%m}?RVI>q{5FV|79phT?Q|M#R221-bVfm`Qnsj|5C|j$f%cPZ z5F(@~9C!HcP|OdQmcP|8)VH4ptXMCyo#$oe<&AppJRdHQluT#QR@gq3>15+0kwFEv z0hKnL4_XX{t!|~+OQ+>rOzQb1@Q;uufx&Tiv9V9tdbA0U29ap^1;?q#=%8f~P-H~{ zPM9LNr={WOM8o9I!7L|6)LI7XGvx%?TV15SLN90>99=i$i?r1D!`a&jHO(1HUJjlz zC_0-#1X^N&{3UXH^5$wIN7wY&t?#bWtiiLe%z!8C&FbXoa%+uMWV=^XxyESquMSTb+HQI+E~Smk8Ac@NQ44IONGL0k znW<=pYj_a;eg^J`FRLtw+he&*(a0T;d$1T}snut%*A`(?mRm;>aho3S?6-aldCfc&M5K>T9& z_wEd8828`nsq^u zxSE>Ye3U}sJY`ZUc^vH;*niTfeubv&9c;zEx?g`89X-z{ZK!oo8m;4F?p+`Q0{Q4;g~6;CUylD2*0#A3bUob(69a?>V*_ z!{XPiV)_YOy?C&E%RaRJ0-C>j_>+maZPpkGCSm;?%;7SXC^v4cTRW0^4CZFzXzA$s zE8vbfr8*wz*S(0w&_ne|5c)g|Dh+KNowVP3K)@*lhDp+^EDj?<$F4g+JJCljR!c3e zDfl|FY|lGdZZIDyw7PAj?4Z+Mf+rev2fDcntS2(yiGBxox8-&nFAx#1f`QyT4Zxl% zS>4=#*9rSn4Okrk;%cQ?<`$D5)#*ehz%R;4U3lR6^KM7x*$n;}nZ+b2PgpL-Sau|Y zN>G718WL8bszg#gn^Yini!$>Ga{glg{lnCqBJ{CH9p(A23fCd09fv%W{KpuYY`-{79CLS>kUy^KLHU z6nLfW@aP)yblIk+ruh9XTyn&Ho9H2)zrIKV=qed|5*(Fl{q zOi=xeu(a^YVeVw49R~28*k#iD7&Y3-gL)k~n7IdC@Md6*oR_Dpr;iPA$Y9-;DNlf@ zr!4UPE`|K}j^7l6N=BNp*An)zxZ3B#E`;C1Tjt=E4U(aTM)ly^af8cG3{T@UnwDM1 z?;C*@f0@SWu%u$8dn3|`y4cfK~IxnjdaXp`H z@(#y==pa7#T0;t{BUIi2V+b?g&+Y57-I@29-p`9}!b5W7A3k`nlmgro3qH?}i6Xgd zVvNm-w83pbgrU%^`s)fEzwvV&+d))T31^nAUsH)N<(3S53L0liJ&qzg=d-usjWIJt zoE*${f9TshHqFH>nklxB|9eHKsd3;)AX-NNxdw*B>0}_9f(omPO;2Anc)pbBL#rpw zPef0=A1z(-wm$x@7r^`sa%J?%r*SlRr=m0T0~PaJ8tLiJxr$dmCTTumnyO>`T=+C! zRKOp1RFx=sCmbG=MhnT^i^8bVuYF5TFng3hC0VWO{TYJDj{rhS34^ISJoX61fdl!~ z?leOeg)?{?OAdYi{5t-|Q?v1R?G_G&W0dOa@7K&1^B!96<9@lQJkG0%!N_#A)ND$V zM9YsEzEG}yxlAc*bo|Z?i_D3| z@2hv2B0*=dx>7VpaJ-}%y~ZIu1<6hG)DQVXyTEm@*z{hnux+u^Uz9@FMM~S*yA7MK z358d=AaOGHXF3SXvV(rn>a#^6uY4mJ_&fz6D?DE(RM{EO)U~|v@$gb-t#Rib_4u4@ zDrKW^Pf3STfbdJKTVUsXwZACf1CC-C%Jj>n6D-jhVflh)r-1c0y%KoO!Sv1Rt^Y;z zAvfb8*z}a-;gom4U77iy{k{y$cLw-&A@^Wk18ijq8q$L0!<*2}1=YZ#p7o#LdL!tP z5-t5m!;6S+6$9ae`{r8*$NW-SJZtUns>lzs>!vK8U7pM)t;6Fw3&VW{u_dwS+c6p% zc{b{2Z(lX26SZCJ`^6+myiZ>v`2W`|2nQm82t_yq2c(p>1>H)0X@h!eObMu0A1bt^ z=~Ax<-!FrmF-J9G=}`WCfV}O=w)!fYGra;_pWH&V$u2+|q@YMQNj=?C`c`8es*|@l zQDC}VS*}@({BUlmrqlTjoM*Tmk(2S%)3YebRz`oo1+w3gs`~<#`wZKy!jR)ebL3{; zByE6rn!0gCIt8v)t%~MXyISRXAM*gf(r^0rBFFNgN9@t%?TVGF_ zA+0^aal;FP!+(85VlIG_atQWh0Wm$~F(~Y*r7GCtHFn4%qX=TPOP~i8U1TT~* z;TgQQ9?MA?m6fDU4<<`Xy1?f&*5^!C!9i^1+oU*Qd8~#0mMe@3vFJmkMv;u&TonUs z2#b{ei3J93v)|>(Z;-@_0~^<^(2bY4`awtX?wc-?N(TGX_Dk!7sq-6g_R(=GA2Vt0%C zfr*9y5$3ZH<4@e@Bhoc9|GZeiL2VNsuJiGpsT%Y(7T&%iM-c6Y&`ToF(9ooH8Z%se zrW@*M&*Yq(TbZAli}u2%LnXnU{`eZ(luSaW8()Z1Od-{^J|F zDD?jxxq?jTf{0=vTu8D`j3ANfBTD~xdwBQU%EC--r^#pBz`?55VkWKKL+Z@EmI7km z9i*Ct?C04LyhDvcfy9Ov?BILcPxPIX@5vxYcANx_NUKiPy#WhL%dJnF0J%{FnjEGn z3>Q?$$RX%!f;hui$iR`xR|nJ->5)(*to&nn^!-rx5UVV5hViXMhXJs1FtXJSrfvKFTbNvJR3pxiYe5F{#e_CrB)Cn+$?7`i)AeOfUSbi*C(%$_R`}e8A57f?E8kGof=z0vPLuSo)!bJ z8be#g>NlW@Z|#6HM5^*1u+otT>`r%r&bG}06+~3Lc27&T4W_~I5T!c;Um{Ac7c4G; zPRL#eUhe!DA}%Krxs)6ybkE zu;NvfG&c+isx6Pg@qH93KeJ~cLXS{`ms;KN7^%Rg)(d#a!%_q|)d2!>|2z(CJ6{d{ zzP83vQr~yLdJtazj&-=N@PitPC>}GUBw%5)4H5BH@>Pg6{=f`KYvm0)SMnTq#tARt$+V@NELh%B^Xgn@h;_l4HeZe zm-bH60mQtWAHY+WI#)C)qo!5XEoCjufh^;^HJq-sB>*6GZSFe>3b-_g*5H`LkR6QN zW_(?w-0Xs71ZE>-ca1W)p4^+obp=1tBg**ib<_m3QhK=R)xJM~y za?CPA)rO7A`RZkOiV5T_6QV!A65vMsK&(YRP}j57*VJ zhFQJ*%+4A3`Wcsn2v1oe!|PkSky|4n6Zaq$zJCijsT3S&InJdY`|~=+R!l{L>$5$k zYbYyOtXJ@UnJ^W$3!+7!iRER5v07#UQR+$Du5#XKJrk1fNj}Lk0NgQlOFq0h2(96vl*?c zmK0-11{in?fA;hyIR}a)LVndiEaXk)s$*-C2#?+LGw{rLQvIl|k9x5iJA;br8DU&@ zXEB5v<|+wn2zY^@!X-^tkn)w0+$epkEUHtI5d`4zFry7O{I{H zd9tg+PZ}+TRNMP-@^C8#peEUqQ~h$(t{J05JnebIS^IYrfJQ;0qOngTB43&mzvK!e z<=Y{~SNCv;3*y3gVpJmha9CCLt4(2b6jcb9%vLcl@_egfP$#lKaLZPUU++)^VLBt< z|L}iIePvjcS=%-xY(m(SfYhc->F(}sl$MZ2Lb@C2?(S4V8Y!haBqan1C8f)6;mkbm z_hbHyGl#JEeXn(1XI*+(F&%T!rrN$T7d3`&S|2kH{NJ$xHZ!Ba2REZXeoPa5+dE?O zaZPG162j)!{^=?yu*J|P(dmp+JMz)tf|Ji?;IAn>{L6#!OrG&A>Aqo<8&SzEFVQq-V>lExH}Q*tpMgEN zs!C5o1f(#fgRw)@agnzQ(TEw{!bRqqjYWELHkS2rc{bs5Ks|hgd~@h&sEtYp#+QpH zCdPI|Nm1YyY{A|SNo<@&KDq(G#AB%)pho8R1;?#mbz@3)}I`puzEXTCu_@=v!< zSp&Dat2159S7KdG_P;fhYQN#XH6?C+Z_NBzCYOV8?B@UQcvlewjO!AiF@aw0$KmZp zw%zZ#l8vW$M8arLSPrCp+}i8U1Pu}%!o5nre-v*n}4jc z%<+(}Zfykm5`X^lLaZJ>Pz#z1P0&y=qzog6(3wMQ!X~mNhHmw2rpWq~SX7-$M%xx6|2H0%R$jfg9V!C1G}(p3zOyaTF&op!eZ^BP3Z zj;LpbQG!s(?Jk@EJXxH~V%;MPALDM!OElOTD;^k5wRS z_yVFwr7x-+bQusph$Qyf3_{10)N}%^c@-CA+Z_bwON>Wx`6JVz^|mc8OP0ZOBEw8Y zIk(y2??o>d0oRk$LY9{}TK z{W(9y*$hSlh8Mbb6a*QdxI#i@Ll2jFUfP53B!{tQA3!=9Y;m;%9X~v_@OJBpEdG=k zni)D(RU#j5+j*E#7E{6_B$yNL-k{KIEo7y9^x(@I#ew(SQ5stW!JaoDZ&}B@F~^-` z;c1>;{RIuRm~v#XAIP4=cbAB(lFiz7OLJ&LJN^0N-LDN$z7`cmFbvd|jx35a8hz?( z^mj%HVSrdcV?if!oILNQb#Nd)qw?nF!${m&G5m8+0G<+nv{uZYOlMSRJt!|j4`T{{ zhq8y*X7MyI^-K@72kXk%%Wsx()`88u)@?VZyM@)i<&R)6@g*G1Z_VTZ^A`k_>cx>R zv)?um{fN+S1|Lp4e{6)kd)l5(V$$n{BVUW>0$~RLf&co_@WY_q9AW{zuud{VdqHnf1}!wFoOH! zyEr#@6G2c58C$8*E`o5-tpVw~Z%>H%L=g^QTOU?TDVBQ4$hwECv~Nngdb^Se?#)3* zNf@%<{govGLmu6mU(tsX`4WpJ!BLXo5;7%N`5v1C*w!;;@l}*3dV^kAGcGWM{#f!_ zuHvKs2v4LFY*n&`m&ggvES8cJj4$$FLqW=!sOX}Gja9TjO9Gh1dJ|yTqMFI{YT`I7 z4@JBsuN!Loo!l2X+C$n7uDYzl&O9n zgkSbcw7I*WpkVVt-k$dgWfkS6cA@m+=fBFT+2;=-#896|s3fn}*`0>nMo8fR4I|Gn zsdyRt$2CbfT#}#{^H<+R=WDK~GjOW-qWKk2n3t)qAPS{_ikRmb-n|5Mt}%EcGkFa7_pfOw zdLCtNW2;g~X{9O9hhkS?ud|V+Q?9Erb8~C^lFUw$$Hv6Waf|uSR%j`NVt-aJh?9X? z=IN@T^#G!Mn4klcq7{tREHex!b4}>O51#!wTd9L%Lsx~2BafY} zQKCozol zh_5|=eb{`mFr>p zsiSt=BKuul+v^yC+4Gy`73yfF5i}Tvnu{&pGo(%9k@ycU`9M#$N5dJaQsI#q4pB83 zvl2Q{^w;Q#eGqwG+Ps2gX|VY8;%fIhLzN^EBj?+v#Y`_Jwb?$c`gxqG^B>PKl~6t6 zm@89|q8)b=Zs=Iejo2jW0{DHrtT!Jn!49WiYIU`Frf3Pis#3Zfckvk9NQJcr zm1>tRT$}6OX=@1E>5IqWDYcpUBJ!JJ#ZNWo3oj~ED|s=Ki_F(bR6MBfz5@@vKOMIJ3+T88q$S3Zz>mjS=<&5c z#xD`7wO2^ulBSnYv)02Z_Y|*of86`+L=MQ|GRgKV$&iJlec0B%UEshIa*MnT{_*@M zjvCLp=X8s4q4G=r|A##gA!{_W7D2svaYttJLRih)$F%mOiz=319vA+6$s92Gs;P8E1U%=yd&w7`JX1O@r3>oNt^5uuUX|;j@K%JH7`z!60jalf*5&9K`e} zXCNfPv0&g$v(+@M2cbtvmYQJ2NAHecYgKH2$oH!7Zg52EI~Th-9A|Up?~e0XRKHAY z`=~FSremI?{;B+2@PlT{;KUQWN>FB&!f|3?^>XV%f^;odFw}z6TL&cLo_`X!#AGmq zQc$t6!G*F198FC|UEUtLC==;W_uFpj998_^NijRgkj3LQ)Cg{m`9&}qiHq`4OG2zkQla`JxRnP;E zH2CK`DfaC&)5nwvX$GZB8J#Kq?^bdo3IHk$PEXIec?lZTjuVxyD*RP~%BMfv`wkjk z&DB*Ok2~pX#ynk4wf}Xkv5yvRJ~DKuB!?E7%&hX+(np42x)=+Rb!P%qz4192q|2y; z)sE$gsC;x9%{KNugB;Nfm6_ES=rltUQ{SXA1m3U2-Ozy!v4Ur3ekE|Kf{s&RaeE*5hV<(Y7E*un{~!HOX-je zP?O79e%hHH0e%TbAsEMHZl*ElgqHwPyI-+>4TOTysuDVJ1H1mEt$Yj3$QQw3nk-MX z2-V8agz)*lF1=bm?}!xz+CtA@_(Z zuuw_JwJ{;uF{=r5VYd{EeC(I>r%R34JL|vG2*QbUnm@lxN^IyI^OQ@T8*$b+j}g&S z+gl<2RTzIZ!2(&$ljDNROMim$<#7ReDP4Xqsk;rr_k4V1SP^NQB_}*~@i>~M-_(Ju zxUA{>D2QW@LJC`&+4;u^cbMTvTfKe48T+x#K8emwVAPXmOvt-NM@OGt>_U?jij!5f z1)N3_fKfEDm8=C7tOM^J=-#9lNLUt?45sVAX`eA#JPYnTWf<|n&s^W@a?MTu@DN}m z3+d?{^|%u`pD!dih9w|lk=b56?RL)-*C@jMh=29YRRuVvS2oW=-U2@L*=+6D=?9;i zx7E54I5hO-74+V8aRRvHWrn%HI8$vYl22$jp^vYJ?Py-)axkJK-B&LEZ`x6S2r ztnl`UFA9>t$bWM$H3tG<2jdv=>0L%D6yKmww%dx_9Lo|#cV)k=`uVNHF{j{Dnp}(5 zCljB6+K;bNo);xbT&Kt2VT|g;Mf!C{shbb}&~eY0T_{qdoe0AN<*yFq*0UAdVPPch z$xJ#A%hu#(vBj}JIyeX{GeZhM0i!(421tLI*=x|lpCQvs17;ECMv=vUVImm~c@h_L zM6k!aDIX;ES+~>Mqs9J*2&k4vFacX|c~#LUy^Gl`A=+Bn&#~2Ki4=yo@*N23XT(g` zZ@b;Hn1Tr#ECFFlVbJcb_N3hCQ>V@#ORyl)F-B12DX5sqf8g!ym#&?^e|ZdAE}9y8 z!DQV0@nCNV?9ed#rZ6M%P5czBPkfO%3ih5!{g9AQZUCIf)w5d@U@a*#8%zimOoe={ z%!Qk1axlugKM4Rh?JIg`o2nDlE!%;C1(#xb*WnY|DBG3R*;8Dpkw~hW&){?ZAr6;F z${K)CE}}Vu>g=)8%JUXKC@mg`fMDCRQa=d1CW^6e?DMxiH?-zLA9E>V4hj`5RaC=q z>u1q}ONvtKLI%gH6&H5YkLz5@bv;^>_a`w-rm=EJ9|X<6hupFxtS=&hE2i~f;fQ%W zIB2kwcmm9@4wg$255c>~HQd2owZoUI9#hjnZ+?woXtlZ&ZTRBkarV=>RhJ^;J(8C4 zzS}Ll434HI3P{HJ**f&>=Q*6HtlDOh=WcPT*$1FvS$~dRBxE;(xb94dr36ep=0npm zs4md$zJI`HsZZyz0tYkj`h4TqA+GKqxM3@A1BzIwQD#5B*pPBP|Fyt1g?b}|FRdw} zeb>6Tv_g}VplY|?ff8O0iW9ZC^#Cre&>U>>{^_Yj+!J3xAT>b#jzE_2SLruDOH@f8 zKKc=j$lH^lJe)qhfM{|NL!5fppQ_%m0waBL#;;_)HYLj06&aH$HP@E4g5g%TH4h~H zN363YAfuH7pn%lVwI~`D0FltOBOGeRebEMZ>x9c&K>MTSD~gp>>3--@Z47q&2EOSj z{ikfE!5?Gry}m1|sy5}%j@EVTT{2Pkt4+}tUE*rUTr1^yJq-cxgrY-~7$ zv>CNpLm`Cphc|%kx-%1%M$r(CEnL4Q=URz%`f@&XQw%OKFX_6&xH}M)HizhbEW@8R&E@a9Rwag@dDKv-sS!KP#X9)xi`nHxcw zH*NQ)i_HhC&C)t^wFS1-0v^>^4qV>5H!Qm|ZM9yNSJX|Jj5firN2zL0^9jvcR)zuanzupd8P{us@23YAKh~3Wq|6 zGTbOGgEtH)y!l)@_k(1-a2Bl+k!%U z?aaE5d!p%?f=cB*9DD~^3jedrh=uxlu48Ip85ST%3N^bX#JfGrkUcR7X5qB@`>Yd9ROIX zL*LZy?H|bF6$@7lX8ULkyO{AoW8{nz{8$C7Ay3&J95U&(quP2p=Rf-^`X`gQNs>p? zIp~CdW|M&cfB$k@z2Pe|goXYr?;vPTA;eJ2l@-27JOu4Qw{K($XjNabdT(kVqk>Vu z3*pQnE(_07>_yk`;nnm>1o$e!15ZqnD4onGm_#au)vKf@p&;s;iWZ7>euvkE{8oj! zDG*3BH9nU+UDH4~H+^Q9E>HUn^uqRVt&4$gaEQbbjQGjld8`qeDM^^Z#cZa(n{q1u zJemLar9?CULHhHiF!U65F1FX3w}a6)vEk%++fM}%-NHB_ys+fJ4Fbr5Gaa?drPjuw>uT2gqxzkju1&U{xvI)>(ynL%$Lj1Sc{~@R&3TYSow}0yKMU?*`vH&-5UiuiwGc$H306z8cl8txArj6)f{e?I%M}29mX0+>cA;JiiY3 zmvIEJw;$}yy2O3_&h}Ib>*i4c3Hi89cpWZz*|S8Xc1$VZ2CTTHyPhDVMftN9KqgU7 zR;sS_u`P&J#e#o_n69YhsJ;BYbrYin}42Jq=@kbiOqoPVt)BMEMNqXRW%u&!YD4X(AyJh%jISjY=3b?zJnu5#6E%nG)q~uTz;#-G0{!`uD}8e z3i^1uiJXL~oMA|2gle_B9_pEe6QQ`J=`$E5cHJ~l!g#8$VcT0J9G7Nc{A1LFNrJaO zMu?1ruNVL80!TN)t?kMSSFCZ~eD}eLJRn*W7*5P{#LD#J9VV)fAF=k3V37;!E9bCQ z=Ke}`!0tL^hNN6xx)Uw(0ZCF>0FIr6(VLJ~3r5{g`F6eMk+}U*fz-R9Z7Q9g&Q}0Z zwM1eo6#@m6ol-0>udB~EFG~)bhF}aoFbpry#kM~^!CKW=7V&GD@dChw!Djfay4*nh zv_W7D`oG*25*z}!-jg}X^7CGU*#|BvoZjM!lAo_EcDz5K?Qkh+yHQN|@M|(Ai?>kb zSd?#nGNUj9F=Y}fFpxYi6u2ysI0(jOr9c>%o&^TsT|Kajruce1*uGBTFD?BV@@uj2 zT^tW?@DLzg1xu#p_>U&goK`3`nP&o3vFabOAPjH`0C%5U8LaQ4aP1y2^)UN|iMhXI zI)AgW_Efy@QQ4XpZQzUsvKC{w^ro}aj%Dz?w7}dsSYIXU%8OT(CRm7pb`Nm$pkVVr z!0;EyJiw5iY%K)XgpSWqBH7po=`9obyCKw62yV)`d4yunKr&0jC+qu;${h_seF2yC%s>)7Y?XFEA-GN&Q`@LKNC{OMD)5N% zJy-&vPV*Wo^x=}!M86b~BT{E+PPlLbE;!V1vD-1KuxC!kfuaqTKvypYKyB5e%+<^8 z=tGwj9MTCW`573wST4#N=R-~meEJm_^8%0{MGUHGC&phoFZ=daXi$5GD^sV0;&UhB zSA^e&k10KUE&^vf$%yvWK2~^_)2gol#NZPC3 z%Rck($sJ1pSZtK192OYsEmKu}#sCDH&l_$|^mP!l*enF1!OgP1#pu%q-=!wW#tjnJ26)F5D|=>N$aZs0yP}VZBzXy^AfgirRaq(HAmBaZ`a}!V2x8k_i7B`chbT#-Ssd~Sj z(m_??!H}`rsvq~?<;2Q?@F4!1xIAIjmj!bIcYUSuRjA_ziU9Z9iYgtnh%Dpei>y*S z%G{LnXMnbhyxkGvXeqi%@Y@MXDzss-%hVwZFcw<_hO!didKx5%f!Kr!x(ywfz6bZT zbz ziLmMNjT8yZ%fyzXaxo2YVPYAJ7GjkTHYwAShS{$j?ws(hj%8bEl0t0%)3y9XsQgnW zfO$KmUiH{%arN<^*SSO-ydGf`-r=!=>FvV0XJlO48q*1Z9){#!it=W!Hb&RP*1#Uh zgdiz*0U?HPf&^9ke>~}HED#tAv8EO%ILo7w86g%DD<|UX&-FXbtkA3o@)B1bCj3O)plUigYszLLxJurwU27?MJ&AMqT za7c!AE_VUz zSD&u0kiY^6Z|OH)Z)wYG=$KacyfUVqFsu*KCN68wRLQY7SSJu4nSR#qFf3i}SUobu zTIQrOwnxfw-#~5kz2nao@=5{L5iq?w zcb`JUZ7e5$ScExcUD|=qR$ z6eN@GiiOH1@`!BtZuy-!|6cp#;_X;Ir1ROb#tTp_91S@YK@2NZQ&&-h8j}zk`7o0>g01;It~rc%b&p=2)`_hPl8hZ8?lqHvO7C83+kTD;%tTs`Y61 z`{Qf3lAa8z9|o`GXiNc>Iawf;y_+!1iVzAxaw1g2m}h3Y1j&U-u-^7@zB?%zz@06ReKtsF%t`{DEOWN}^WtxZoWi549HKc;UYCMbrKE z85(NWLbXL-rjO}=e3f^E=-y0uG0sn*wUq!1U%Vcbx@M)eQbG#k(0!v1l~KR>J&Vak zX99cx3KrSxjvs@GVf=0fRdxdPf$d2Lgm~|O@uv;qzYKtCwJkC(3Bu$r-)XRpM-s$B z(EHx}PJ2M#<52n(S{flzGWpTZbP9|&e2+AiKhdE|#YL*Qkx!gdy1g#G)Ni&tu2-vX z`}fngkjm4kHDoS-uUYeMs#qb@RhIe6_2O19b;zs!DBFw@d0hFak{V@g9?^4&#zjq+ zAp$%~!>8cw`0v{jHKz36mWKwRV3M#tXM-e^f`Pcf{n!-boyi~03-po1?!qZ3y=Gb5 zM@&lho?z6)k1X1!cT~^H(b|5#>uibb5;F#Gs0+|J8MUhj@mUN79g$0jBMeMclYcHY zG8HSND%+hTJY2p3W@0eq(^kn5K}M)^j=W@c9zyE423q|0ft)S*>_AVMV71zC7Z=)> z%=~F>W%@6~eImP|pXCIc`=;gOOm0a@s*B^USzC$C+&qRO@7bppxY70vt`Z5P;$KR8Ab-U6Q+ z>Xgwx$2GVaR)$k|PKFNYEp`!FUFUsAPVmg~0|Qn2=v>DH@g@)WB44j2i~ru_n2wt# z_*d&mq+$iG-#Xs3BvZ~{5F#5`0p3Yg{$Fl42hswV$NQCH6y(P%=-UjOI)=F}MBq}6 z%;8v|^J!U>(;ckSv^~E_V)sGTsdV!AKk?{@18K&&*mI1t^5*%AhCDn|gs*`8+Xeq5 zQ6P$xrk&|~4AlawxV$NJbFD(FatC0!x}{Tqgv!7Nq1pDYNB0s#;boPPNVZHU3GFfy466&iKlXGduajH#(-Nc*E?SA(qNPr^B3HPyK&$ zD*o8Aj0o}Z@paa)O@cnr4Hiine6-q`Hs>!q9>`Q8p5qNg#u+`7BctubNsJm6wZQ2UmcsefX1PuJ4fKPEJviOBSPWVvL4rU@^{eN10F6%prwA37DR|~I{NWJ~!ax{W zqZP=*l>tu@JUTLN5WJVQhjEmsBSl<|trR1xEs55=i+`kIj!OC-4+R^|s4ztQ_>J8( zKmXi?$Kuj^gzRj=&;!Ywk49m!|N8~xMB)A<^*oq5f){^Onzj%-FqYbr3v9ajjd}A% z6>KY=d5XyG!Z;m#8IP~n<+Go&*3=dmce(z{tOH~!Au6QvrLQl|cSqi}P?NXBEsfL# z9X`vU`1OX#P)I%0QmdB9>;(nmH5lAwL8frz=twNUH}eDurao&!RPuM ziuwlF=UsFpi9T>&Y6HGF&=eb7LQP0&xT(+d#DQ*@k_+Jb6gaM{GuQUvBt=(B9mh0K z9LL_4ibo)?U2Rc-mrm555=2#!>NtH~_-;t0)u+)IO&5PQ0`7n?`NuDNJDHNO)HrBW zXGS9M4BIHATz*NX6LXhBE6hI5oz+;+l z7cd{x=&))SN-gyiG7lE-z8=lsN?4O9Q}ed`=&;IM+uz*Gt5*V|sGz%&e9-l58%U@R zgi)j1*IJ`ez%^Pewd15g=uK6XUPDIIa6-Z}KYX5BI|>0>9SMMu-0&~$D+LcN|0Z0b zAi8}Ly^W)vCW;R`i-kY< z@om$jW_rDPM6F-Zd(+}&yU3sa^W;SN=TbO=@D9-)v!GyBjbVqi=OI{^`)n`|JTk-j zPJf?HV))xb#M$KMjd^=8pX<|8P&5h!1NA5K;nuhN@<48BTwezecLLmSjjt|qt3iT} z)3xGzI5`Ywdo)vqh{IxL@0IyRU&JY}{aN1t{}S(?yijT58vlW@c&e~lKXN{HVCs01 zhKOHScc+9tPw~6!;f$JxLa?#$)AzVb3N$_ypb?*|=X?!Qd;JXo)1=vL7vaI(_lLVi zJj|=1c%F-Cz!d7)Wg_w4DFxfXR?W8&52+J^KBI1I`Lh1_Gt7`Si|j1LO0GMJsHgmL znMM%}xrZRK|1Uaz0Lj$w@MroF%RcwUFni;71T5z$ZVNJ4FjbH~z6OLrv1u(hjv`(> z(|r#EU1I4V?La3d=v4$`4B8~-woRk%(|VOwrI#-rBH6O`o}3TeZGVyG_d~W^TPt26 zPDGV+ER8Lw6HLmIMKH8{MUV?Lx=0I#zuf| zpM=ss)#q!IZQi?8A2A)jyNG9RaR`1A2Su`a^&j9(?mj{rO@HYZ{`FtECJ-*5h!Gk5 z^vdAOq}lB4YW+k^JG#v0Jf+!N_Wq2ucY9Nw}p)r?W zXj!|rTG_|`->Z0w6yUWXgnVKYP^FqUG4ksgq!cK6MgCe&3nr6yh%LwnZhic)H<->&V{WyaQ0KNQM^rhhEZ?Lb z_Rmk)g$3>+ws4|4fRFMXe3E{FcaA%#`rj@zm0Ep%sRsHCcBr{rBgB(8r!U#Vkgj$w z?&+NI+E&sCp8CJ2pSOHR->&V$(lDm~HpWPEy6e-OMu zr8bFvD%d^vYdh-h6;dE7e6S_*WRx6%Y;7t)2L5EiVK|R>riX(bV7nz$_PM5ii)V+j z`PX}?&mu^o}-uNZVSSqv2h=2}71+;{;G)=RqZACPG!NmU_c93cFMoN-b4Z-8O z3D1X0GV>6I;d0$9nEoLQSd%aY58BBe*I?|&F6vOr)QIl2wl5xi;Da_sgcmFd{aQt5 z*gRTz@@c=Y9Z^O#BT&jV$+rQZXhav!P0{O8dp&**3+Rfl@8+bK;lD$PF%l92WBeUB zR}m3nfxMu|6VtqMfF1N85x=T+EuNoMP~zdfMG`&X~nreR>!Yj*u3p?O9xU_zDnWuc%$D#SrUpz=g>5tkj^5>R=!u4Sl zVi3X;TI8S&&fhTbD#OYlqEcxeq)8c12K5;~nXqj6+ChuQ;)hL~zc$$yz4FN|^&{3P zDbq{bF|>{9<^S(dfH`7^hlj@nwSg%?6a&ZOV8r4{{$(48a)XAGcO8l{eZe_Dm;BsX zYHArbi8(Xs+_b8ez^l|>!Zf9be}7@xQkI(2 zJJ5qZxB;$BYQ-ayzb^=^1^W`%T#L}baVTf6<2@&W8zdZ*Apj=U08ueG3aNS!8JG9% zhg2@a)#=7$$DpRahSpe_&+UPpg$SJ8?91CQ+aF~FZ`z@=0L zwo%Ws+6fWr02oaZR-MCf_b3SH3R&{Wj8fpgiOava{p@i5;1xx{4Hy`ZU2J|swXXLb z;PL=I*&sDGw%2fIhkr-)E%cD7NGw?njI&B0xPQSfa7X_|_->5Q8N%hc)}I2$I#& zMTwqJ#_>5@jKbG%e(?G|%G1c|0Du+UBj+6cGyU=$j>g0-6eh5}EzyuMIflG>gc;lu z04^V$eKnc}iGVU73l;rYL7!N+?lm=44GPrxi}Pjq5g0f^HO3A)%73Z&sUNo}&};Ns zNK(G-6aw)__UznawtYL@0AS`HDoFCF%+E>tM{Y#lLlZ-B9(|nr zT{5|!*pFe-N$wV;jJdB;8hT3gExZjD_#NDgjH4&T?O zYNr7j{10@^CY0robeAx~H9^s}>rG?37bkco*_*2Nr2f3jJAX8+E~F#wv+Zzx`+IzN zbsuPiI42$d5D;>mG5ZtM#A0>4c|T8_VPmnj$AqkD-Qb@c5aX{zKKPN_<|J zc>rK8lGbk>7)0uIhGPn9q++`HWz^VSrayb~ZiV0bUv|Ii6I2wzd9yY$I=WaCLiaR* zf2M(MO>Ehf|Jx^ll@1q*fY<>l_dF$`NWBx+z&nejf#qfz{HFrZa?B`nKS1|agU4?R zJw{;3o&iD{3Ktovv@q2s%`I@fm=o5^$OwK4UFaWxcfGrI9p^c)cn4WO2Y)mTRJCQ> zh#i1Wk+#N;zxSh37z&wO9Ilpv)1cKSW@BSx2iSpK&MGYNdU#U@+CG|oMq7Muf(X)2 z6R%#KBiSD`ygA;GdI-)3*dA$qV_EYBqLP2N?E24W8?sHv*h4B`lZQ;N)p>jvxKMw9 zwh5vJR~>CZk6N|6^PR~c^&ctN}; zPfEBSr@C7SzMgxhM4iQJ;b?9R>EUM1Q~{XPzEc_(tv%<&Y<@Me86l}I=x@k>1Y>4Dv;u| zJ@$M_VrLPAt3sc0+tS~3xduC9)(sGU^t!Nr%w+ubbVseUH9!UJ`$L3%uG(%eu~jZ!Cco z)T{;MLo@|yjE*-uq*rTRuz*oS2-F}u;r8KNqjN8{xz$p~&$b-(A2Yd*0qeuvq|bT! z!-E6|@A{|PS>b(1G8T3PUS=KMZIJ&$P+D;+Xj6jYofSm-3rG?O0-8HaEv~=TGrT=h z4^*FsFUA;`#3nV$BN@Lan$Y(k#ozv?`GOU}P#Ug&`uGw9KdzEu=bP7|kSA!XwE}bm zr|+#hi`U-DkMV!31|Cg`FdHF>*$ho@z=u&jGl+{8wCeQ%Vd>Z1fsg7=utaV}f$OVb zVFExQ3D6{#FvOf0(iZ@9+$k`1smP7a274h6bOMTNiG~QMyw4BeFO7(UsYfOW%_w;8 z*K*FlL^=UZhy4WC9n+nO{1Zd-pJZVdh_J8Vf%`r|1q}h(XECKzBsJr^qc<=H6V*Q$ zMC_9GLAs6R4|OKnpaLgejOZ^bzF{)*@mVo!74&o($n}60B$+3@Ej;RA&s=M=V~5z; zj{(siw#&`&qR3uTtjuwRV5?5T=tiptFo6Dg3(tiOd0CvK4-!|@PFgfN?|U2w!d(k( z5U3gOl33|C;!7>(-_2j0U4cuTQkfj+%+Y<#@&E5w<6aLvnGQ9xvA2e63MG z%^JkJyV4w*`b9K!)l(VFJ^wGs1(a`kXJgOe#6Qs+`TgZ}(wCJ#OHPT^9z~Ao4;1C0 zTgr5>qNu4=YCq5964PC4R!0nM_5jVvMN>$Ue{zJpAdv+09q$1bZ->D4Nc^)FVoR|F zY2m?x>jB?f^{b?ejKT>LBVT*4sB#~0d081y7mHNa;W8cVgILLgy!h5^gOx^wW_iLX z3iT@RYZhOdfhCj=EimPmhyf){0Hg#)LQLXKKzN>}Y$&fYxE1UBEg;{hG2Pd_e$BHz z^l-Ev2Fyoqq^& z9*H0wkszr<$6ckzlka&bzY1D>&a-iUWA*ox5sd5#5@=$u_}t{O{x03+W}c#SaZI%s zc?*hU@`LWBV;Ng1^t<@q8POdR-VHCveIZQMI!!5-MBM%AN9~jP6EFU&bEf;f&hlI3 zm+hR%w|g@2iSvH7C`O2A#iRvGU80xs8fgKx>MSbQdr*^;Bzd{3$Li3{CwN+{Wnc0# z-h7TbgrboX4sg={7A?_bs)}pd2R$mu)J2^>?IS31+m9vaW#bA}IT;%ZDAP{wx>44O z2Z|za>8!0uv@rWJdypl+fT9kX{QQ_onsBKyPO*qCqI2;jp@!eGuKGZNM7|bs1tr)(b?4_zPlcD zmI2cpg1!%w144YlTVZr5o+K+BKd|LV^52~yPd$+=Z>)GSj&^)m{dkX zQpYPd#^2wy8M$u+&}vt@*1H^TJECJQ4we6Kdqja8f!KwEzMF2?e{v$Ihk;Rvl8E}O zD?&OqFXlg@l-CQX3(bS*HVH*si7_@Kn*&2)V2pnqKW1_yd%I$TIY{qK$cnX37=a&= zQXT3lb_xT^*&hf#?f{kLW0u!vlEgg=6eOjS=+<#?Fl@1`oySG*creUh;`=73hzF@%k8g^P$M0h<;#m1LW;rSt7haN{VMbYb|i<5I1)>g zm?mSd0!ha@2~4q`-7B9(%vP);l*IfTBdyzE3;jJC<)caDnIIUm$?@@V#);a&(F8fY zjc9n!TBhL}iO3mH)k|G+TTaFs**Ei@;R4KFob^}5QtcgWDTa}Qpb3aD+9$F zxUf({D|ie&g4G!FT3+9Vlfv&ST_(TjEDL+Mw6Sme%w>zybS_pZt~A#HZ)aIPQO0MUn_4 z-IwdcjD?%uUdgs?UBUidIA9*5@`r}9mLc9Qs7q}gSzaiud%ZsJ*4+5LmBjC*ta15W zMS0De03$gRE?$U_eI|* z=^YL6)-g<>W>N7Jp+9bI3mX#cnrGK%Q6*obGQbar z;#l?M9soS+5 z=1EkN!*u84e|gX=eDjOLj~wmhq3F*y(7$zcjc<@YBRm(Q+m0ul(1lps{vX|LoDxba z9*IvPls@FJtl)8tMJvN%MBZmWkS}XF@hQ4;Vo=OyXD%Kw$2I)rf7Sy+1Ry4**nw!= zkjIY?Uwr0Ot;g7Qo-AJ4vD{y=qm3qxSHBHxtji(PjO3(xhDjz!??#1^F?4LDsl2Zt zPiy$KcsZ$T`JUb6-vqQ?Ab_-1B^*DG7u_;DJ~_RKLsF- zxj~!=5E(u!#qftash)L-$}X^vi^pYGb}p zXjqsA2*;hZHMj<2N*PL?9-^xw$SZCd22=c52W^c7R|Q7t0&X=Qt$b=uzasT?@UySg zIHggTQJt(Vu99LIU6A!eDY1atcYy-GF_Jn40T0^@)61CCVErKUYr+5u3L;-=l2?p%0m;m(GA|%brlXI9Hz0}?h_@`- zOh5ThPF`BS4R;5V*#fiCzH2#YP^;Zw0!_kNpAgAe5~B*!W)S}0Xu5Kibo5i29sE7H z;QqjXiTcj-Gu_8zg7f)KIQt|bz9i^Ym-f0B%LNqe4AfYm^u-;w`OA^2UPz2qCkeu{ z@sc4u98#F@o5NBrD4T>0_)k_{l)btsfR!GOk>{TR13GM3?5FbBc5MUWjdqzkJ-53q zphv3BV2xLJmr18kKTvmM=Dl7Z(dt{K-(r8iIYRha;1oi3FBf$9Cisa4St1yL-u%Zi zmp}4VIv>y)zGxJU%YNQ|R`Nc<9Kr6jFlnrMUcScWFHUx-+Vv~>t5=$HugNtd%4)HS ztu!%jPFebZrt46tTW~k$(OyISyOO;$A+o2@>3SwU)pa3bauQ~$>5aZ-=^>D@>^I1gZG7D55h8}44Q6=R79wUK~l>Rub37e%h|1A;dVi?Whn;xKv zJwOZ{K<5SSsq{mvE2Nu?LzWxS*Hs{rJLq~j2gp-NpP$F@AL3YYZ78DId>t~{zW+{g zxOW+SHFEuXZ{I5i`%6rxg0!pP|6}T_!=h@pw*d#Gq(PAGmXPk0PHB)7>23+>?v_SE z=`QIm3F+>T?huLZ89nEBzW17!>+%nDX7=9CTI*hSRD#1i`_cMDeBp!DlQmah^(VWl zU9Q*iT}6ODEUxm-oO?Syw^=XKoP$NDBQG7gfsKgv zxDNbi@Fm$z^&^h2V9B@EyiB`bKP$jGy6qx<7ks9Ok8MAas%N-iMWivbYY)?yIGO%o z`mkX=7-{OWwkMjd8UgP@^aCMa{>{%>AWk~INz|^hq7%DX!z;^$N5-A)Rw5Z6`^Kbr z^n{iB^|*fth3U_cl|QA?(h(q(+C|4_JpYX?boQB+j(O32RoEKtLB<4peSnrqB?B)FEsz4 z^;KyI6gQe9%*^m)H5`s__4rxD#PZn7GV13%6sZiTv%;#?SKY5-mHX& zlgSdyHs#JH{50XNYYfX@$q64HTerUr^0qrcnMd(pB3^~QJ}uXU_3fOV)Emjm41u!i z;VHYKTR|UFfCX;8LbtWIx4-S;oXIiZbKXnfM^zcqVK%tfaJupo!?*$z)Y*%BAm$Fx z)nlQpK);$IuC&Cc($qt1`rcuye`0d*!+$}fs}4t7eN0moV=9XW?2X0ug z4{h6}eY-;*na(S-)A|}mi=h9gt5i^Gh{^-rX;8o%N39Zc7|9bo`QKEWK<-GA@o8FZ;w&IkwonfSrd`*5UB z{NbH4iPZ};0KliS*cKN-SI?fHcsiajF*0gXzrga}#i$*UFIY?sz92!MYOq@q{koS@ zs#H`W%G&>d#5V?=p5KPr#C&|vn@4aD6Ry9!zzQ(kEjWH+<`5f_u0pxWzZx!4V=h2& zeTu3+kL*cKB-ml-S|T-yWgueQf&@1+=jKMg7bSyW^OajTFe#Y31sJR1wU#plYhA%K zq|Cn0QOJ;|tSobHZ@uDa6sH{olIS*C7E^C^eG#g6egq)Lm;qAIR8yLP7m$d|bDk{+ z1_xU%Of&SiH2`kI{*vC(5peaL7VL$KQ~X_Wc3}?h#IWb#`;K1Y^HIgx9^lTsQRF{` zcmZEYr~RZu_NPIK@eQvFH)M@mgQk9X4ioT{jFf`A{Ee5#891R^e+ue`hb{J9PfWNI zMBd7cctPpv8^ZuU5e{zi*ofq!{4w${Lb=1U)(X{hx1lRtE!1_P4yE|{NT@h^TeuDJj+xpu^2NR2hy3&N(rZl`8P*g+DaGP(|!?jiPayL?# zZrUF270_xh$sIn$3#7lZ zePa^nfUf51%<=MCpZ4R(`|qt@)x()gr>?)&zqmV{HvLY>H@6(5IzumD`11=sZaB>I zKU}kRYsgbnLMfp7PF!2f{Pf^nPG`R8GWq67Aq8mb`XN3t8bu6; zr8@n)@9a=}+oGSsN52L{r=!U*N=zed%=A2ymPz9*e$SdQ2AS9bX$gR=*E;Xx1dw)fG=apvLU!?+pI3+0D__y;?7*O+((&XBD^DDdno80x zZ(LORcBCbtH&Eon!gaI9^%1@^!?ioOhpMrPpEOANCj{p_3Uh7Z1d`poBMqJvS6Brc zH0eZ6Z^szdn7ym zhWL!{W$o_nJ~^J<5bE!Qr@SO0Wfcv}&>zj!WTG}=GucfB?t7eAI zNPL%oV{`CUY8=Q>@+4@@QHH`6?SWZ@Fara25aJ->SvPsV&~I~q-%A9H%h`E$1wGe( z$L|U?+;jl*1;hlf60I??IGOeYzkCvW5if_8;5@1C*NMsrF;8gcitQR8r$(eMs&(8^ z2q(R*sWRep{=x=!gi}%BW4^c|+0^v@M8H-Lb!reyZPlTJi22bj&0yyf8bxq8mZf+n zRK_LJU~w248jE|1x#ah!q;^dgx)Wh29(hs*+Y-vJp^cy8BIh)se6=|ck1JARzNM<; zr#i)Df`KG>c#Dx|;b(kgC6~%hr@V6W&i?(!k`uwMXJU_vnBl3kLleL1yz#VVU@{>aeLnPCcOS zXo)u%SSNoy;uu+e?2W+B{k=VKoXAOvRT>gX<4|H`6d@6FNqzQ%uPlOqOChQr9K6fB zJ~{ws_RSU(7AKFn#>q7u&wjGe23u{U#2-uD)BS|m@WVA&tgksaQ+~0Xub~SKUZ8wE zF`XlrE&d~cE`CDrMn~kw_o-2tE=DO2ccOIstk-y9pH6U8G$-RlfbVl+(#Y}4bmY?_CA%WapWZr*>>e!I<# zQm(f0K}+yI1h5z^a$LAg#Pow=!7cp2$$)kf4PX1{bZfze4>ivDG(digYw1f{ReW{G~Lv@2TFFih9g?~Zg+uItes z*vRsV2!u$9xV@+VHnv?SB_KSyfs`UY6?^~9p8A1e*qZB;!Ixg2J|=!GZA zN!%#}#j*x4wL7noW~k=-};^0o3UYg}pkOWB0*ly5(+q9jOgED5^}LK}Zj;939eMu8`}yDWwaQCqx`ii%1aV)Mr< zJY{v1m$?+gdQ2&-v}Hk*UMwCM)n2+Br=wC|9=e4p5K|E;d$Qfk6xj8C$T;_$Wc9Q_m7ASNJ+50&*H@#bG} zk5yJ~4ZgXa9BCyaxrW+%MEG!Fpg+p>10u@cb%4n5mrRBpWKBi(-D*yogXOi!W2tO< zIP1v$yMqBizGoeau?R2BG3c;1uUUau4;SO*I5i>p`!7^L8XKB6+L3t$(C&F9GFe+{71j0#-N^S2QM`nY{~dXPWhxb`Lnp zVb?mtXpY=(k1nq1_xJO_n(X!quHvF)dgJ3-Q=w;QNJvFUC<5XgLf-RyJ=}OkMk5s& z(&xiRnhE5SX)8D&!3H|bBj7y_ZZTgn1DTyRAv3{f;{>*(A`sqfhnMHc6gvNNBFWeIDnHvR^kS_rA zSLXusZKouIe0Grh5yMAfjCxZVBCq_{L}of!WhMH7=#vB6e6D{bxWd_FK3X^ZOOP%r zyZj#~!S)4oO0y)37lqLP+>8DF(QDp3#wBRcMJc!=o0m<6jm`|XPf(t>hbIrK@DyAX zn}F>rB2Q#CoY)byCv&MhdN><1fkXA0KmXC2D?n*de*ZpC73CqTS}hkFD#OI;0qf#7NIlF0;s!*c zajjO%*@dh*ydUEynIU*#Aq-%lJER%@3oR)zBoe_=NWRi&Izpz)gLc_f85P$*sTnj? zU*~T@RLCSBYuPXa#W}NL8Q48bzgeIy+G{TwuF{{BoPVM3nZqcIVTyJ=O`Af)xCbcI zOd%w}Zi%;eKh~)_*Rf166fsWkF8)OQ3DoooQGS9fJ&a>|H#?TObuZ3Wp&g)EhKARd z9Ou|wG_{l6*e5eS6uJCNbIqFqm8P{``x~Nwtd~WY!%yAdLC|xSGP)RGSWq{(ethI5 zsdo7Ia{2V~ED#B2JigSs;JtU$_TjfjQB!Zj76&rjQNekQ>w784m}toQO~=J! zXpu(|lN7cOL->a!|Iq3+UE$4m)c%x!_L*?oabi3 zNWL3(Mp?wed*ZzhPpyz{w#SRDFK2BUulHk{M$hkLEJAqidppM<&{$l`M&F4WI6{JE zBo+$4>E&JSZ+cP(=NvulpA{s2W0z$uEwevoF`?{!12}XpOszbOD%|U9^gOy7d^@4C z7$K2SN&7;~F|x0zr6&hAq@$ROcVAK~hX2c1KNqkw34ym<68OSrw{&i17}|ohiz?=Pi+Ap_PXqrE&Ecca!Kx6P-65iD z3OeH=1-8%DhlWR)N7tvxadvb8pN0SxRyMEucX{6VNcN9zaz4S+5iDZjqvjDg?dg)9 z$tEgpVf*Bl*GX<9?WwSf#iETtzZ>Fwm=-yWbdv_lH>2K$57W9u_uWc4e0mZ^EKrmX zuDKZ;H$MfjdB4_)alDd3l2GwsP(y>DzV?(mzP|X=4%27uJzL@jo7F$QrlfZsRg(BS z{cRk#%{7{pgUy;3HZr&Oys*Nd3H#3-0V2KH$cKCnNwmR%){Msv}T{E$6sL z9s>KUhZs@`XNX}MZO}!c?@V(@@5Q8h)LA>mWLV|II_CV{)p3Pqbk(;+?f#b)yhI_6bfl}m@ z)m^~qHXRi)#s>@0;7R&}x3FbLclPjo7Ue5bjX~g0raI;@JX62w<4oWp(mDK$u?2l` z{>PUQ?Ty*c0GLDW)gp+E;3_8UvqiU6*e4>JIVmhie&gDX8koT3??yz#gS(;_yQu{o zy#*viSiC@jqwCULl0*NtsIpo{QRr^Yil^G{ZoIVF5avgKn@hUN=hCB%tGQ^2~=J71$q8na$&V!d`54uSUAFDaretufc4=au`?)X(;RPFL@7tX8 zv2n0JGmX1GbfuD<(CF#wye~I?)2lA!1CeS!M>L>m!2Kg!gQqcGs>5~3xQ-fLg^z@j z=pl*f_VB!lwQKQkImz$X%PJRAoh|`0h#%l?HeK?056;GYt#yL0%uDuIs{W9_qI4&hc_8%4Z3_dk#A) zO(7X2H@e1MYui8{WGD)-v#B?DUtNG<*9saRb-k-qlFkBW;b<@h7gi+c4#VNpycKXS zNo=|Y33dHIAcB=>39|eZMB}naSEQ5 zo#lnBufu?fG-CcHaL{8652J7s&M`F)LKcJueN1L$Dz$lg!Aj-|iy)hnAGJ=WBv1YY zLi)>FOfvHDj9k5wU%g{2o@)+rz^#;0Q9&!~j93rxFti#ySR7BJJ;FFrv!J9aD>P4Y ze&;XTZELx;K3VCobG#&tO3R`zGO&;zy=nIQ`^{782=mYHO>jwJLCNAG^O2x6ZNTig zSnGX`B-%~m--N|T-t;}4;5=6M!KSJhZNhwGXd)*VoABh=#`~*tXHYyPjO&-#?q?lM z)XP{C+MvP7TgUsNf0xnTM49mEO?|4{=sg(Z)Jk4yz7U0S4MY}2z(p6nld(b#oI+!K zM>j$xygag%mRU-nSNyC`rVmoFG$D}7=U+6R&Natt#9HLa3lxF#Rvk-Je}d{%0h!_>qwlKcSL1WcUQ z$S(zL7P)J&upWlvF==PeSK#e2v$Z&p^ms41aeJL~b^r@mArHGd`n!OPNDVSowlW<; z4;A~;2S0v(?q4=IN>A18lGbU7KUC5T-w^!aB8aYby#xi@zQo-A=$l6@u5s-_S7M`f zRN@RBMNF-YSBL$6bfk<3UgN?oYRj+ioy{M8_gm{V_HfA-jWtBxts?kur&%pyVZb@} z0@H_}9;mx$713tpf=x$&SoJ$Qq9)46;?3Pj9i*%41GkkU2V0Q;TJTV6P))D6voVi5) zS&dLOaFhT5+I!&rIP=+| zDqwWUj1~fkMJ6$9fUXLKb1`F1= zNTNUegDbPMv&CE1d!FOEC;2UrAVACuu*K2cYfN|7vG_U|;caLm{hrx1m{x*H8yZBE zT5-n~&gDat=~gLQm@Nds4PNh4iQ5qD9#^9%nf%dPd~JwN)AgC3@lPjS`&5#qHhZUu znVnrlDShFH?5i$SS2NQmyO&Efj&9Ppj%sDnsOj)=@}#mErg=Zo_46*G87LixzgkUy zLid&)iF`@4=ACw6<=c;c@J*Kw13mT&yDZvd?F@;AGtQTK#_I_1S3@LOI!$ z4X*R{>>MZb+-g9E4S@A?IJr69kS-Hw97>Pa=(v{JljHu9suy(-Htz(N;-1uqwvdGTB+k9!g zztZ6#5g1-pq?I+4Mhusc718DOaOFG%0ZKi!uj_tR8h4UN8555%?UXxC4DrMTe_rP* zB%Mrhj@amOg<~F6dPNT6gh37AxHN!73h9-c-oLIEp4JZBW;_N^L^kWGqiCXa+8>uuTQN409wA2Dv#A(g13|E{w*9oNNd?<`KVrgRxS zzHFlNR1S}|?(dJ@q`y5HTGKSX!rJ`|%4qpHmTA5r0Oi73IMSAAoiuxH4dQeJX?N2A ze~xtCinS?+>O6tNY82=-j%ja$=&sqezCMEIj08j`cE_B;IC1!&oubsZf|Ip@*AX zJ$bM7-owSl*RgY+fbN(FWF~O%3&UxlPbRhL3s}^1vzLGn1wTH4?K$-=G$Ks}7fJ{} zTxNGn{G9z|0Dh6d=xhDlZ1%5!-uZ^`$-K}TO4)G6X7Oo|ASk$U(CzT>{p>CCz7+AB z{LJN6lgE$WAIu9I*rU*pjB}yLsqa}MXs}LS;Y4g%Wh``^4A5lBp?^h3;u{KP(d+}8 z06}GqZ^g0>dpS`Q$AN-DHMcdvmfDel_jv-oF_J~Hg}ca`BSIcqaofBX+qe29sN~!8 zIKzG?VvYR9ahns2j1Dzc#XY2Q4Zd9#CLf?*KlC7VFa*J>X8@8bp5<}tePczh0lZI7 zM`LAA>X!*9KV@~qm0RlDrqJ)CAi-&t1O%sSx`j$hxR%I38*a)~SP0;r7bNJVAfZr3 zuv)zH`iLN;;?|FcqfJ?BTw8-XxzJV3qHAY-k4L=xga7BQ$!}@!TP=|7k&I%L+7mA9 zH)h!T5S0TNC*YSq=pbYJ5=l74abrTxVy3K=@gl-~D499BO}GA4_jPDyahmo znxG92q{m<@MO&JIxRB^8#U5XH4n=Pe=i;Sa1FweU5=mV=yRRG$Ni62u53;{abSP^? zUjOv#(=H8w*$5GF%kD6CaQ+#M`Bm#d1==L~Kz<;uHY&SHqD3PCX;;0w$P1}q&2hex zr^pv1Z}PK|Jy*CP;M9S>PT1j~nk8_@(As{ea1xoNaK)m%ICD9QFEae|0VRo0UaxNc zX~eUNBYy9@8*zfeQgx%J;0n#4Hvmk(nGK!n=?~Ft_va{l)D~qu)wy3xRu6#kp{N0| z0t+<^9-%E%W(oJQZROwG9OrcoTF13iUXj|G#3N`&wAv{spdB$idpR9r zIfN^f{$w$7R!Aj@68b${lfj^2Dc>>jVs!_fu>WP3W>GVPhSq8W?iEgx z^L~NLl_o9&uz2|a(D+q2`bMMOA`HGDpy}e>I(Tgv{Ur2ACjZLoGH`RBM_81Aa|Q*qREIE?vs3)< zU1SHR?{%3t@A>Tdb{4xH_@YSClxcT}k z4}y=+2#{Urs&~*X`NR_rim450Ir+TZK;0Sa)ARt$SMd41w9+8C;=#k>n)AAPTpC?%FD~E{BuUnL%z{VCRBxm zkx>pPW$Gc;E`SB!;Vi?PK0uLg?C?gwl+9_F9tcV0J$w3awiBm=+7_6%CTbwWnn14H(|bmv{!z>{QMy@D{a14 zGwZ+KoDYN%p&bkAgPuK2W9X!mFM|_~dF0`~kxX^Ade#<1dNo3B%EoB~Jn1Ds%T5hd zumBCt*zzmrx8E*?*sikyQl%%ik-Rp1f)%E6CS)oZej4?J4Sb>qq3kf3&(&DEI40~>QSA|;46 zJq4iP7R!yFiW%=#K@Q3^&wtm;*(07*z(!^I{&{04BVUJQ7|P>au)sCx88vscD&4y<_i03sc9w& zpX#g8^tckeyQ=qz2h5!HO{lu?!16)N-7}5fy$b5&tO7(WP5afPprq`e3wurjThc;A z`4pB83dZt@y`wx$Rs@O6pQ%Tn6NcL|kbyrGQT{sg`L(I)pIKNVdG#_~x)9|QOy6A| zZ?{X56@VVltxN*;Kh9DM-MmEmAV}z-oNs(`KH1;j4|!+oyA9<%3?{~NdMPUXT!*t! zWc`#2bTA!myC>(9XDp})kEFo6z)CJmNK#Jn2F|rE@0pn7@IdT!r?T<*g1u?kE#b}Q zRtTF~5p{m+JKES!hhx|Q6kAtO!|KnROjlm;8ZxE{DTd_PvSQ) zhW>h9muelndw`$A2q5Ik5d7ZdHX5204?c-ef1cD@jFs$&fpr+Mc#xxn#QCEmjR1vv z)h`Q7+^g>(-(8Q(B4$p??r!(XJ!UrF@+@FCVS=c7C%>|K>xCp*1p`00`5q8TT&inX zXFl-;cuc0Wqg?>1PeL~Lq8V`1eC^98Z|n=aB#LLfd)ccUX=ibn^%wz=*=T_#Vs7}; zl+JEAfQZ^I-i+=IJnfkhNm9@o-<}t@(xocs)}0wX%EX`U?CF3lBr9FdPSmzx0Sv#> zubADNY|kDc_1tjP2|rXRCu*^0D2g0wzdY8y+0R+@5jD)e6gt-phGCSI8PzTz5|Wjw zP33%s+vw3mXg#AD<@{0Wha!an8hV-)L(Ew=Is=tRt7ym;Eg@Hv0)8 zOX=bLWs)k@1bR->fkhf44!vp&NTD49vRLb@4mt1_j8eedV~X0e`)`1I>d+m_=_~@A zrw6l@xmQPzD?>uM7NWbxOurb$S3p=%qw{{!k1Qd5?RuM1UD93LPryy)g{%ohf~^wX zOQstmBct~KGF7R!d54IGrih7lH&kaaC36etlq}5pt)7j5W7b>bfP^F}%laL;haAP* zjS*hv2gq9(XXBZhu^SCzu2F8ES6(+G{vbGp(f3oO9bNkmAYf~MpxS>abR`{HjW<65 zhJcbO$BP!0{e_ey?ZA@ihL?T)j>i-+zfWmuBxd#589F8W3^wW@N0Mk9pR^UOj>(lx z73b&&`Ty2 z6<`5Jt5K4Kq}YO)&@xR4{u9Avlt^aMDZqn@4c~w;*&8tfY)O`ygBOQgpMDR-(5AgF9p?JCZb$w8d0T)G<{huNUv5KstmqDl?ZgEgfopPHEgI^Ra4o z=MpD;Q@HD;!}bKMUq&eQ-;4JK&h4qFr|n&)vHgIRvAVY7z)Z_0ZP(qHhbqAZVVO}q zI#?b@zB~cWzd+r;^XCnmCNq?MrzS=2_Nm>GU|MA9osI1Zqokgf$d}P9)(jb6ZxZQP z#oqAfwzkdV*P2yaHi_SK%Q%U!@~LQYfbrBI9%QuJt`f~38?3H;0JeMzb;5vr_kE$~ zCPHYgx;3|nIWi*o1V5Q&x2+fs9*NdcpnLB1#CKwTHzm3Qa&q#A;)_Sqj*bqM8Z+g5 zDPLr1*@0A!;woYWHTtzObjVWemgrf^q9Gk#tm;AXCTRW%>@X+li*Cs%Vzyl z>g&#tc+?T*L~ z3O&8O)boT}H44`3cOVLw9-gch7zgP~gC^fP!Npr$b49@cG&m#2{UQRJHZy8S-Y11O zW_PszS6V1n&EOa`%2b1JG^0BlUoI|b*;KeXWtsn6uFXH+x>(i2G_9Ijp<(gIzPBoR zZ#&E1p=6YsTZ>M{M0GYQr24(Bj;{Eu_kKM$&{QU}J^U$(48XAR6a14bOQ9$?XbyCv z+o7yLuuVt?nbr(X%9`EJzb}(68Hg8teDY?IPg4OzM^LBk0Zu!7nAq!l12FNc@e9i4 zlfLvtq-Bi%aTbx$?qCOM&>mVVE72g@kU=%5p)#td&Mcg?H3slz1=y2+Sn9F}CBiB* z1$HXLYpmM9GUrNFUL9ZZOt=pUs6C<|;)?@Uz{K?#O%xI-`R((l!E;E65v~5j!XPwyH?vqjEX6CC2=&*c!%Z z{4}tZkdruU?Y7DU{W`6%$h22r1yif8YIM9D8~(PpU+A((`UXuOS&#ki_q{3)SzRY) zeRe8$2Et2*e*MgyaK43KA)xl2!)znS+t2>NJA>^TDEN{gP>@hMKO3K<0E#F7edVRX zOfzBdBJQ&yv~AMKbRc6%4iut&PosywuQo{@UtyNd!-mM|4_}&w4pdWp>uzIx=2&Nu zW|2=i|9lKP{Cs;j9`K%vIZQN8@_g-;K_^0}2txp4&=6SL1CRR;2gZ$%w-~8t zX{EHb4JA3SgQXF@l?(=)!Kk#=JE=)*K7x7E!zoi~)U)~7G?SH@XLB)@pM{jRtKtbg zc#Jsvt_K;)`@td*Vx!8`gC{wj61=23o#6kGC)a398Re9yz05LRsfCU{ppi0x`D{4DFo7692-ZQyx9QYDMVtX3w@WcqZ!sqpRBLC~l%L7m0GXwiEGMXu( z*>VdF_PLcs?B?U*2GSPP2K9ib?mTKY3WOa+i*2G9p$(Cs?TH7FCQm|+AnHZ1&zO>` zu|mVAikLuTo$mhb5nvGQD6_`{@Zyabp`V;Ciumv{inKHFn3CRH@q6LFDGkEg#fEuV z?*0@ynD@R9ZE;*sS02zT@FUF>nf|&r$Jht$)i9-A!D={CSr9$~%af^aOz-Pc{y!6n zLWQQ(#-D3=+rGaqOMf`YsM#Ai@`gLZjfvAAO*S}m_51&dVHC9wJP>TRKeZGR68a1h znX7P}oP^Am^yIqbc}^8{LKhpSYW8FLEa)tSKk*4j@;AIZO1K#Tjf;l5I3QWX1D^s~ z2GI%F*RkMAmm&xY!4ScShIO!5ffeeY_G~HjnvuWrHzSr>M}I#wTwlNdt(+{;db{+? zYOd-tC-RU0Qcm5XncwKf&jk)wD@Ep@qSAjA%BAR0sFJBi9cWq zvjlT3o!SqFFKekGZSPO(hec)_VakYj zUW$r}zMrcyk#YA?q{W1mnj?9bO#9VLeD&FruqWlw36ofM_~9ME?nef&`b&EQ2Z6oh zVyl-o-7OEzvhZa67uV_y%s?6Nm}3&0%vh3O26@E}gYTssEe2QbVa4yGkeOaL=W zN=1Ro1n=G0tO1i>vFPb6bPmpNs+_xaaMw|U3FrDK$szKV*7c=5j6deqod;$deIOR6 zD1WP83?^Ke4ePg2M1o#$A~z*MG^r%d7#NQ24#k)KYVh{aWKAQ6*^{ahEeSVR*I3Xd;F!*IT9N5cD<1B{Pk0ot-; zi}{MzD7g|(e6=4`M6Ju`;XThaQBo0I{7FN~`RL(UNoXa&2Dm}7b_=x@z&?Ay!x4cd z+MpFV2-`Q)8eJ-j9~xWQPS)F?nzh_cij|rZS?Axzh_IP3{2V{@Tp}Y@jUT8qdo;>J$eZ;+>&u^HZ^b6!N+C^QGGcb z*GK#z^#li5P8GV8X|DA9|1<|a$>_p(S*>1wiqHk!dck5M`Ps(D zr9+x1dlS5#Hx;+a?`ka)Gv8lgA=Lo#brCZ>uF#@d_67TQ;55on4E1T+b^;Jg;kFlW z?wD}EXM2x+=l>g63{e^PUbwEj=aqU!T@aA-r0IILpnuBPwp3E_V7r`w1G!T%&WY${ z3|~1ECvjl6r4UTzAPa;Z83$ymuW_*Mz7 z>xtne)7Pud#qyGLHyvNs)5!V@?WLlGH*=t7^P0#gg3 zFVvxm#jygf1SsdgjdP0{wt>))Zlsho56^E#QQ=7wxvz8JM^rf6&_D z_p}7QMGbgqy$2x=7F^@tDTF3wgaMvr&n8GVkK=bgb7Ocpk_PI*5MsB0C+*Awm!j5F z!3NI`>gSpX2B+Mz@}n+|R#rj#5dWjZ#axc6nsearNWlK6={9a(HHXE?ZY z3pX}BmQP+EZ%{O@98kmGqF}C&j$S9lel)Q1{ZFZpB5i~KThe^6PGWQ1mQ$+**6?f~ zBz*wDg$`*Bjc3=2!2_pd-2!p z%vW}iMrwC=lrzOdl#hH*x`iga-l^(jkU-l`75VlI9RJ(RHEnF0#MQ(nG9t9kawz4{ ze4j-S6ekaE;m2&maKC0ZIvI9{Cx7@{mPu+yB6= z&j9TdQ=x@T>?e;IPG}e6?sguu>vrzsDT#DJvOP`44MylqFdilWOjJ6NUTqU#RhI~X zWxLOf$q^gqe1H8ibKgqTcn@xhI0lVZ`PhtVVQJSH&U}S{r zqsQ&lNU4MSHg;p}Z;gdC?uSQ)WGLp;Hg&&?bo=Oj@)se6HVo~j5b%i5ka%6hk_M43 zstBI-Q?R)n>C%BE{RW?GFm5?h2J_e83(ikh~hl(f6Iw@9LlOe_cix21xG z&3;|n9JnDC;pNwzb@eYeSnk_mo-Vv>biIfnRX^}AURx(fm-b^&jg?CzPP7{3NQkkB z{#wT{P4nMrf)utGQsIkK2{EMl^_6Y_iueJTF?rOCOxZaGX) za5MApCXexOn^v`*z!^r5c}VIIe_!F>3-}pgv)o*m z;1fcE;JYu~pm&YEpE&#NkAsD<>(gmp z)F+t-*WeJ_0`%gg#kupZgx>kB0hjxX$~&XlOnoTO+F&OM2O{~Dm<=YrDkcLd-rrp! z_~w#5an7FvqKwuW;T7JByZ>tau(G6<{8zJ~d z)EsO^kWX<3czCKv1#3iKj98uZAIJA!IiR*aQQFE!k zN_z6#vj6vJ@K9I3&U6@S2J|W}M!|5(wp*OE`#u>59$!$H8S2p1^RuH&{x4hnKM*XN zF92C&ft6eyz_u^iUaLsc8REfc~m+wVgs8mz|ZOO??59p)f*1AoCkC5JNJHK zF3$bpEMhJ-g^>T8%_Ud}{jsuPo_B#zz@&=*G=0j?{-?6TZrbVDDDu?S{NpyFZroq? zLi<~Ub}y(kWuk08P%+X;kgZn$VU!9Wxopu*Z1Vr_UK7S>03ymT3QZ7gI|sO14kp)K zXhO6@INjOj*rBt{dZ=yl!e5wEf`9g3kHFAzx&V6Rhm!<48lGtA3Kay{qY!c{0jQDG zw&gMjZ0U8rRMRWAMP}m%eY&DeiY3b5S#xPG{Y<33DVi`bj zhrq*2gBgDlij9F*N3*!s>EGpt{&d= z3-LTb$>I|m@#YnVu%jTm%p}2mazd zk}zTIs6@Qq#&FDVWZWApKM=b2u_`FYs-jQ_;|~4%Kp?xOAsB6I?+eLRXSXtDZjVs~ z6_0l7EBIS)$_oBGa67rZg0)pa*>gPwhF>!HAt`(@fbT(HuMKW6jY`)Ghkq#LzfYq< zbE4$Y=Z|RycvlX&amc~rdXydz6g0@S;+fk# z&Yl07)8Yl}cg$L3EaZP(vKs{~iIC7heh*Pe!-wR}sde;Qvr~l9f!ZF&&8*w#fvw}O zGU?kv0sq=@QBuIgVe`1M0_3_T5M!nR5&-By?Nn>}24ELDO@rKj4;1vLPcfEH(*-!O z7{YOA;Yr*#F|2zGhWvyHl>1AsTV0$GCcwEt%kT#ou zlTV?Mr`hfFOAfXo=%h63bgJ5~2cBpkl1>7Dt_hSRPGOPA?Uk`^umH*d{SPe3))`r+p^S{__+C z*&Ij=6M@Tt3<%l+;*7vl*P`yD!bST`!q@-pF?>uhH0y0LpuhugXL|)UUkTt?M&>8G z0|vivmLiJr-yBUh?b4oNR+^XYPO{CEF_^q4NkoAUPuccxBwy?&@1^AI>2Q940uj)q zA1SrAoLmG~?%JX>=^W(b_$!#72FU^aMeyOq_)<{on^MmE`W~bx8euTOn-1ds+bQ#> zw;u{UDzUyiA`&ojESSsmP)0%}?0E4iEg?l3?nt7Y;9Y3yQcgny_h;ZCM~bSbshL8r z0+Q;tjIdv&+}+;~7qR}Ua(`?4cmPbM8WhAdqI&$jlT|bdI`4iUm)!#5D%@`D3-Qo9 z%ck^Sr$OC|4Ig-0(o-eVJH>iK(6XlFe&b<{W(hG1e7uMOF;|6~z=E_N;J5k8xo>)n z01k{gWfDGNA~WAf2%rD!gVQiV8tvbWSPpg+Be%Xd z9O&Eo_2dk9Os`uo>43SeLOTfh@yZmzFP(cpW@MlnGiE+JT(;2Y8=m&HW(wv+eVs@R;R3se#OlKh;g1_&}*NNQjdd!>W&j5_? zKCTX?N=qH~%e&8zV*H@gcE`7Vq`hl$j;1=6LpBQrOX0&iWFMAP9^s!{%Pxxtr4 zO#ePF1X9MVPr1{Jng}@lSa~!JV|v&Q=9NTe*Tj?$a}6H!umpNBKH*r|Mq{l60U7&X zWQqgtL&f|4QbP^#;3bz0BQP+F1&!$Z47%+9Z8pGY4ck!S@9+U2$8eUfq`)|tSPw^Q z-b}V^f6?RU&}>Q2=(_2HNERy>t(pw248l>m1;6FwcwjwuK;8zkl9cU29aC!#s6T^% zV?+zxSU@SXa8w-NbNjY~i~1hyKcJ(C37H$P*}pMQeDzKnWV94<`im+n*rS07;Cr8! z!woH%38=wJLirF(9~!_Zb^SzemK1u0+!$aF=9@33cEI?e%GBD`#$5P;@s2&;z}Gcc zYGm3rEE@OR7u1f`TgW#&xd#EiEmGX0iZph5dKnHSU>t&D3CCAl4Z6X*H(LLja`q7x zn_Ly$sZ#{KzJmjr#Nr`DX9@_m^U=i*xNKwj{?%09_IR#AQYEGUYR)nRb|9$pIJ63@JQRQ`03G8Hffoeu^JO_oMGec^Re;I- ze@tCvRFzxT6$I%n6{Nen`W^#kH!hbs?gE-VnzlM@cNNb zy%Y;=r1jcvcs!rocNs*&y`YENxi{<1roju9O-rlUDg3pG1YIm@nqgG=Pxx^a-S9?>5xHCf(!0+?z&B)6CxI+si0{#&`6O(!wXW1k6SX zqu4Jy*Fl?q0FqhilEO<5Ha7+tX2rP%pTMDb5WmaxMJOkMTix&XQCE$Rf(>NeY^@}U zgXbxQK9A5fh=t$52^)bWnGS8+p@oi92at|w^%m~8&vbWB1pb_rp9fU0gE4P`_uGL8 zFTkLS>a6uR4gIM3`S~*k8{Srq;m)9(WuPC)A3jFwrIqY8T9I8HVi@#veln&yQOq3;hqYUa z`}j~==EVO;NxUIo^+GPNz!|RCu7Df@Am%&h8;*)0X=5NFDt}X6h|dSFb8}^+qOp?e zzx!E(((8vf7)|ve-WzyTSJ{~dyldmf-0lA6ut9Dsy&Hz@wpW^ZmHBO2_o!;)&%?vz zgy`W_Ia`;#Wn*N&pu1I5M^a`p!wWMtRc2eCJWZ&g02#0zEJmqry#11Y&#?)LB|1A) z`7sp9hOPR=_lI3S*oHDX;Oz1T@ZiH%Jb>!eKEsRZinE&zFoLb55#l{4e95>}&Wj+T zbKke~cAzS^`pun(DiO*(b4;!~(rPjTjA}kjhr6XXW*+!U7&MX6)HgE&StAE^cLst+ z5j9moKfWb<-hILcI*fwaY|a30F^NwDb5rn!Mu|xWXmNlR5zucVL1TjOmGOf4bLl$L?)Id2EtaLgSBXD{8J&3rh&u3pgA5n}; zi9=Fa39E@EdC`#kE`sC1HmU!bA%B8_0j?CaZot(UNxT78^yWa1{(3N7ukNXr5lYj$ z5IhFz`ngpAXm*Wnf%6+}4?MjT-Cf`&C7uiC0E=ydmGL4q`OHw?mK#1|BXDZ+(f>po zfy<<)_TAEAw~-|c-gYU9Ly1OP}RaOUg-jzuOw+{g|rs_EgY%lePw zLRMscClu4eA2o77f<5_s8*T>*XKTUCkY8e}ng3OHFwPF*MCLz#rxvjmugsbeA;YNTnQ(4o++iW|j5K&r=R;#o91#SZM zOlLR-TD|*wE1UxJ!$NB6xr3L4rKU;M&!I;OYFDigCJ#7($&-FS95Q14P@AK{h`$*R z53a#b5>QccSxzWJ^@HX^ncjM{9E%J`|Gyp0;mSR}0`C)0@EVM~AzqSlb8h%Z@Sxbn zFh@u?K;@ds>$n@51?QDDu}UuHxWstJkuLp8`EMIOgPYc7Lxg{V>ksePj9^j zw20-f5dBEUwrPWq1RHSZ06X;eX8unvU6dQrn>kj%xipsdi!i;i{ zslaMgx*os%q+3_XrbR~vX4IGcVoe9iGsO3@lMH}U^`dF)1u@Dq;KHW5D;##5lbyW` z4TR7#>U9<0Q#>D6y;?gj$;=WqV?!!Hv_~djF4kD0xx0xTB6R?gD?vecTpve)%(M@J zy3PRT>f;ih)cL9^oC%m!!$Ki-A~8@+1iW%Eqp;10FxAwgn4g?jzZT}?RJB5M2LlzL z#FHoxBKf}9{HNZSRh$PN_?xFrfb3EUqzWq+I;J0g98V0D>6{q<2scWDjMNR=HT<@) zyP0TC`O75#D4~pzD*p$bYh(pwyT#odzKp%EWDT!DAAx|H^U8@8|KY0{Q@VN;%#AMB z>+2ITuN0L)Ef-@qS|9iT07@gp2E`wOS%35^q+LiRR3DjWq}k2@6yM1+3H*d5*OUitirkqu2>XCrix({{6JMyZIwkX=o<0*d)X*U z(G8ta^!(gb<2!oiS=VSBV%`q=v5^#3Ph|!Ye#?4$h+42;kHsyM^?rqms$Xllw5RM2x<<<(CMkjgymyt7dD)EMegrFTUu}6#@YuQm-5Hv z_Jh#&q+0XsXY5D_?LoY?>|d?JgRKE`D%xBd>DYM6QsNPTTSXm4=Oy`VLl3+p75Nt7 z63w@6vOX8Kgq)xhdzqJ6vioeOoe1Ai)1CDt^5#+@pohA8wQ&&_MpHne6^nIf33kTW z4UH{ZupgEiT(6=gc`_Y5u(Sm|r(jWC3Rcd$pWadj2e9P$UsWFQ79uy4Yr=%&Y~!Vg z@0}eNFzoDSkm=4AWO{z)R+~S$TY?%)FNPZ^`|uekn>0e=Cdva8Tr?Xxs(8}f-a+;) z|90Rg6;4Rm=G}a!h?uDUG@#>bmwuP0%qlz}oC5d(N%f~^guk(VA*5J`Ml-c8YgppaJJ zP5)!j4;_fUCa>i3Kv^6#9iaU+E}2c|Yv9{pRFf1u<>=we1IDQ zL>6Z5{ZIOEsq z9ATB=(x34um-eR_YJb12%a7t-yIoTayWhSGup=RINVzg>wP0qb%SK+VXKc9+qHx!+ zShPx4UgzT>>8&wY9Sbn3!mi2_&XgEHEbb?hrf=oSqdUCeBLKu%7{Eb`0j$ZaPGhkm zSn0goRPHcu$OY#9V6+-C#xoi|?FMF3|KqrC2q#JC%VY6qcI7Xadge?cmpzuO4t^F- z+=sv$EPzCPoQI|fUkAw*Gij#*b%XyE(z~;L!*jp~9myK+P2wnc^%G~0kk>Zc)XEt- z>tALlfZbyA3LsEWKpl`3ac*1yxF#_Sm8?y%z64V}ObjFD#I7Gvx5aMzCZ_OE7N4f5 zi%Y}d(^69955&D<`YQPEs1PZTN(PHcW3*xI`1$(sW+$oEMB#hKN{2nooAv-qF@E7f z4)^+{vxNYX5u*M16EIUv4p11V!1Pw`H8oIgrkq2wV2Af2$7FUTf4k$iz=J>uD zy`XOp^a$VWCOwrpbs1>B!ARv`vH(ir zmCe!5^d7vr(VHK`JU|;!MW1K?D&k?f3Z|l$Iu1XT#~pZF3+oL)oljvb07$?l0=P{A zNL=ZZ4C0r3xJbEk+l`13V!dTc=(N7JZsj6PgN{SElbb^JU>Y|Q(V-?29oq)NZOdWs zR#XNaLWap3m+sMm7%ubsXgjLl0Hok0q*quEk(-(I>c4}29+WiSJ9r5k8x{rH2g<;I zd!gpy;BL!~ICW+hylq3DQqM1KG1M9CQG!CNZ}NrDau~S2;gVf)3>~#uzqC4NREUO zp^`3&_vZqg0y;UC&*d(f>m>!blZ<}^EegW&@RR~f zM?il2+0l>~pH4HKUqC2UhS!5Nw2w@8ym@CK=#v95h#3&351%@NX-aqvSKpFfEi8bf zN*$r={Zbd&3W5hF?z zICLn0>I>uQ%l*E85M8$#6V%6gf2C4;gW47;91P&@PVg}yi}eFJ zpDIe3rh6y(Hkh~HmQDrL6`O?iZ*(;Hw|$Lnr_uEByXOF*`&Diz?uh!7 zQwEB8t#Dj}GfZ#YTQw-0vTMBj0IUJ{5XqBT%v`86U8o;_bTJvI$!AG=VXD~s^F)D4 zy6>yX{@;bg46YPogw`?W|KHMw*iu;z1{`g&D*gqmjsx|l%D8PI2>4b9NEYa{*zYuP zzI`g$XYC9^v<#`o@dx7r=f=!R&4puz9{uPKX1#;Y zN%7z?Q$B$-akJd=(HK#^Nx(G{iui{E0+g0=i4H?~7cq+Kh} zBGhkQ%gO=;MybN5z>&ZoKX%Ucj$V+ zZoc+Sht#`Z3+%uqcTvSZ&f>e*RoxC=qEx*yJ?8dQ;tNIq{=3@{q+q1%zFg-kfE_Mr z?ansp>@>9v=*YY5&wDoj6v++{lKP4r>biLO-)|RQ_@`undM)Hdk2xRhHs>&WDMhiY zu5M~IWB%P~tHttuu0|YlDDN>m9iSK)GTbFQ4VD=lq!Kck8R3An02r$`itCH!Ipf*U_l7r!u@}+ z^r}Lvtv1387SJn~@6-1Vfd?1^Bj)z>&UjgdI}oY9MJSnlq+baJEKz}bV=QK*ph8Kg zJ@L&TZ#4ZJXb%6>OC|Sz1#D_mHMlIsVgVVyH%~qx)?yd{R((M4nE4)I%lM}VbPzaH z%)=}X7< zO9gsq6_LNOoinYFk-r;$$SM4-82XUL+ky1KmW4cuWeyj2@#RM!pAzkN?e&E5)3}2) zwkri%b|*2X)jyJNO7xzZrmdCet7OheyJK*OE%4oaYcdw7bu!s@NpwA^rCm?9>7Z>& zHBdC>O&>$B0@I*ovvGh3O#Fwzd-NVh6q0pkOQD0zt}{H?#d${An;mCgr?&L;YRq?-$D><)J3 zJ0{BB2_hjsVUE_BRgNpTUqT2}=R_yumZz(?-#w+S>nx!-A) zsEm6mgL+x-C{8VoNUva?zC(JaEYlJVDbB|g7O(?q`9JlsS7y;B++}y6D-{CTyj2vB zdgIRLlD}V#S5aWCOCC4at;h*dEV;O6hsA))j&58KR75ZHUKs|o zt32g;O9^T?EX^&^@+$lfPxS2DJ}MC`w?6$D|F|0fIchqRI4h3pL!$E+N1%(Q9xT%F z>k?1=R*W(6mKC53Uy> zmY3iA#8v*0GRx<*FTOlR-|Xl)QRZt>PQ-BtxIYx16Z!3A1b~CR;|53d&aNY zd6ozohn1KT)95mQ){zUI=oiRWX6Vow$#k1|y}aO-%%tTYex*ys?j`R*gXrqG@I z=`9)^+q-zU``+*);qGJE5Kz7lKax+Y-c-G=h%gq9t>-y?k%fX1gz6IJmX zo{ry@z67BJ+sK~>iklMd@vl7xeZ~ibo{H$}^jL)IG*2yoepKbnE=$m_q}vjPGLj$m zopx>3QHCdDLUW4c6ehPW{*`U)vd0e7|KGR}SFE$~gx9Xe10Hv03;YJZ{2;5{Z)ML- zQmAZdJjgvf@{Majmx{>aoUe}6{7cIy?Y~@JrsWuMdeox~quRYol`in-ueidk8IEv3 ziD~(Z)FCk)yQHVBqlSDDuo|htkpO5%*EPu;AlD&`b2RllVWKGV zA7aWC{L+NbyuK9mz>uI@sZo~tVDF6hr9vO;zXgIKcDg!WA>j=w2H69tU--Rh1K8A$ z5=C#*)JS*0*nSq>K6UxCM1(t89i7=gtCwk(HtF*?_Yc#%yZQlAE)NX-qv1>Ra!+XX z1wJySuooc~P1b*7z7nVM29Nn{xHnG--JY=rNF7qreMu3QeV<@vC{M11QXpo>HaHl! z+FVjCIO}2^vVaP*rER)lSHMyEHO*5N-Esx5aFq0~Brb&j6i1^Lh3Mh^56tC&0oicR z++}-i2|_`*+HUVI@s&WDbpP)CduS0rk}%+Bk)AW7)L8vsO7NhgCHt-%%MFKs$c|ir zIaZa{G20kZ%VuCeH{_X!OUR?=?!!(!VEXT7sl^1(sJH&c_Mav$au#l}1mRn=>5S|C zx+tITgwGe^=X<|9ukXIJA+)7rr@1d{)p2E@N$>q&D%+zSsm_nk+x(T>JfI}={akXy zGiojNGKeo$IoyJeu6fB)W%*Ca<<$pmuB=!iAxPZU3mftfEsT?=X2Y(^oLfF2;LGL+ zJ7@2|%>MzS8nXfkOE>@5$V9^NXf7N2M=^dOe$c-rh4FTy2{%h2&qxSF`;3#jMF35a zw3yzwjnQNb$DDyK-0d!BhW6SMnk9)y|E((qOwMs8P-||G?rx0v;7GBbWYUrSSB-BX zwy02Tr|~QUN}!~Xam*iFFke}N=1Kp{~k#@;YJ>% z#t-)XIBw3ZMbk#jfBcLAq*u;#pk0O2b8`wsmqU);}M$qLedjwYHaJjK}(B$fxErLdi?_dn6Ug*hShJevz#h zkk^1!LXGG1sja>Qt_c05&wy-eIur#fAKE(&d|8fc^EyUkQm|l8}%Lv;UWWYS4hM z{yaw(ek+s#H-oOPS0(X?zL%92MptLYcRwVG2g3R_80bpbA0oPI<6vo9rrxT=D>1Envm}0Z&Q!CGsOY7L)%E8n!ghHu><(qUOIJ zDv3y0PRywD(?PZ3bhkIj@b7ZsYVcZXTwlJ)e=rwiBqt1s!L4;|?~P4NY-$rA+cRf4 zP&Ib6KmHqLYDUDrU2!bEHu- zR>?oVe!-5*{N{gNxGAjl!5iDX39>qNQ-589Y6)VgX$R9;qHjpR(91#&&>rN>+E*7p z(_!AP=&mejF32}(EUQupxnxS2uUZ|j$KwG4?P(pAol_vY`qc6H{{*NOe(w+i&nQ}s zatIk6n`1igvW(U>sF0EPt!ja5nVDxaB;&gVHpuMr`OWA$`eo77H?Ln&XR&;Z%%-3Y zw&{BHKey^{99#2{)=R_g<>`qCN~5KxH3&bDC>2OaHHdHfE@qSJnBqQn(B*s+9Jz-M zL&2lt*hCABewwV#&F{lm0YJJg^BXt1wE^{|YzsWnkkJsl4RU}4tiHtj^#&{7PIuvX?OULi+nyAu5zq|1y9`)P{^BK3qy8lt`D7JR%|~5LCJz=e;;eyeVzb~ z4D!R>G%Cm!YyBlwrS3vay8a*i^7hN$shI!O?>SOeIzSts4CQzn5kLcew7wiEu<+zN zl%n`7C+gcscRYhl58wFbveo={v(#YO@`bcQu?cWI>(002Y`zhsmU(+2?+aR`xBR%z zbXppLY;|xF>tS_bON|1}A`T|7-t#PM^gYe(tOe1f9vem*wf@)4yrxXcH^f&jE;H6X zBQxrULQxlPf=c5rwnQGuc9_jI|C|bd4kfDGNkFdu_;da+Myax--1CbUfceDv|M`jY zQ(6D}XW^Rb{_ez43 zz~qeNT1i$M_T9}19?g`jW${$fQ-ULiy&(=0xX9!{OmMMXi3Wz5%r#C{afL5wiqQw8 z?U^(Vx6Yx&d-c9MD;Jp-A;gxFNua`(T`sj!M+p5vDk+je{RA1m0?Kg%X#^mQ>O=0S7uWbQBj=23xnA5r!1VgW7ezRdgUkcFV)5=TM4hZb5Ff=*}7Q&4L{7> z^Q64(X|?Ez=Pv&xv9%Z=lC~pG2k0lVs#LuXS#~q^S{_HgyN4lSuox6JeV@FAmRIZXkT|Yydb{Tg z05OvefUB?n|Nhv24a$@bo24BM#a7SG!;Jeq}nnzs+-ZCFf@k+U?d?)<8G@At1W^5@9S*8EUj< zkiHkSKwz1(9)~tL=nq7k)BgN2|Mbg50te70k+}KE-T=Hi&(@Ghx#2y~E>mJ*Xgqgb z@FQ64ZUA#da52~b_27%&Vi>GsR^Bv*06hV9M8)&f+pb)sR|>Wlp0kz$UtpMs?O!YE za!QerZCNP6REvA~u7Z(iu|Nd1K9$C5zYOP`t`&c<=hq>Ljpw(SwhguI-vlbVvz0Y9 zL_!X9GN65}qiU7~r0D+%Z2F=tRJc9_T@f;0F0uZEgr|~E5^|pg6MD-^S-d+NNEiBy zF47_E@DNP)k-JI+wRn0F)@8t(1QiZ^dF4-bK~qQ>TngC(849rj_sIV#ZsB$;shOKSzA zTShZHA``bXa0V1~Mh1rI4lY&N#2RkX>^)Ljw>5RXiR{u*OgLyIQ0KQr^#}eGadPRI zQ&`m_|A_X(ah!4L?z(i z;I@OWgAp~$y+}zLaY(%>!KK#cIp`bqO`xg=TdeZi`R)-a$LTZ>O@5%E{C4z%uVBhg zH0ZCDUK>KFmvrWVkLXrYxIP9N8dI8)rsm{1iJ*{>{H5!stKb3Uf$`LK1R+=_5l<92 zzYTL4otmz;uKP@H3o~+m!Kz6=z-yrETzGk}B`eD3-Rtcb?)Hau=xC*P6qC$+9NK|F z>8mRzhsSSc+!5S2d45frLr;%)vJ|OdtK~MK0|Rn2Hd+F4ev`qM>pwJS+*XDYiWK*o zU_=#mlpb|8<2l5-UsV2Ncel`f?SXs>&Un9{=#epvzDnHl84g}JL!RmUZ}AOCUH8dD zJ)C3$Z89$yu16rpPJaL9*b`#*BVAE zPj`NW?TlA4HEgOYdNjHp`=cW=;Z zhlhucJ3&DeJjI}(QA~xXs9*<&gvfW76#EnxGlJo60PM7c0O5EZYD!vKSdMfAInCO( zxYr}6y}T5kpM#;LIB0iyilPGK&_hD#T2Pzm8h!S7ldzO(ZQo?@dpxT-@Z-<;g1VR6R7y(_ZFck(PG%;B;`6r+Z!L0;OM*j3DWBU$y@*kv|gF zeHm$+cApU0j=>eFJ?;+^C`#F%bniI_^rs;}t+dklYMI2XuB%Jb75*#)V78)mv%sq9 z0U&fEVqkQGF?RbvPBX=X4O*2?K;MrG5dVMHzBAUf2d3o11sH(n8w`NGg`=3(Cm{OIu5*5~`t<43W3IQcU9!G?K%BG?Ezc|pq&;7eIItuK z^MSSGJy*iuut|YvX7&2dcA3ic+;S|fRv`O@&!*KZARpF(UZqsm} z%8k}=FVaWfMstxJDm0rTX}W^1G`#~H!451}$kg_0{Yd#&T(B&V_IFkF+jkmdU0{Y) zkrg<{zKDLntD+k?6Wgv2k)dFGTwZ<+lu2;FC4tAfZUsNLpgHKOFVQ@en#2)*k(r z#sFTFHQ=C~f#r>gkEec95=b)(Ce;A46$YdBm+!BIKmP?227$c^Fyb2#WGE40?yyi! zthS!f{-*BY({~<2tsn->3@RKpR01JpyPxVz2Z;rN2V?StE8ut?`uo(h4#JXL#fOAF zgRWOg9-@rd#+xeq?4M^zwx`D&x;j(!qZ5vOD>5d;+Ty~Tcdct)67Mq?4#yburYm|l zhxOvB;adcy6fgl;2&DBrkl6->UvrE**B)Y8p8pJ%3lUr0d|oGf7krWxTB`UH3?f~% zom>XXS<`;RXMqiorvQlXza;2lFsVLH0b};!AnnD2ukV(m6qaLfu1}k9vVdLT8KAOZ zy|HZs(=PD=SDoS6C2%%#opn>f`H?#e!~sP};fwIJOM34*@d5(_ zhXm0i`!9jozPe)B-JF4zcBn%NYgWR6gC5)C2srC7qG{3VJAs%g0bu>=@>KGPo^$pw zU{uMuC;4Pw+l7dlI-b%DpKb3`LAn@d4RNw5`Hlg#i7{XaO&tTaT_7-Z5^Z>9 zg?BK*=>iF^$)A2WDbW&#nb)Qlj99docLr>;CZEbxx%v9ST3ah4+n6_J7e5aS2uw-P z_!Ne4cfd=PG6&Ky%xtcSUEk7Ho^B5@)k0iQi66hGA%7$iT=-&Xes`y2OZ*@D$NSKG z#6bPYbx8=(K`h7f414G$<1I%O>BjgdAf3p!SnoG#x8o@WWGTpoV*BDZ>hFKo5DfMZ zj_pf*{hLqB>O0sUUmhOGIx8X>YE7PS*WTEfni<%qEZ_HFi8G)Xwl5JL? z8%c0FZ68smC4e8%24o+Y*upP^5w%ulJ&Zzvf_}n5u%@SDX{lzNPv<NNiDa6_}RTyVe&RLYjrY5xN5#gW}_tD3m4 zDcgX8IxYc0%6qFXfk?z@7Ww8-`8=oN^Gn}>%HDVs!Ytb1U*D>R{l)ije@5P=G?`sk zku`ox`AxwclJ-`mGtE}goJl7z?kSu#lL2Uy$}eCK+hW|^;}d=^SCOMY1EVUpx6t1* z>DBmVCV#Z`VtrS=ufOi&axK4iEY}>x>bbOX;rgURtN(kp>1`SNBO!g}epB|~t!hn$v1zEJT#?!!0H*eNWg94sy z=&l#Izy||QUXQ%RBo~t53SUC5I55K@8J)qA#XgnI*x!$qmY`7Y8#f35YOWRVr>hrudl!N_Wn9N*Q!ZS zvl+@1H9rCarsN_@Fep3B6cxj(C(Cy?HjoY^iZ#m84Zl4GcTE@we-=B~(bWw`xX8@M zYb0vvbOyDbw(4A5J~t*zeKxjYIp*lzH0^QNE;$$3_fpBWfd5wKl2U^^UQ3W=&?OL7(LGVXUISl))I7@zm>l{UN0n8 zTB~kaaQip9Q4sw$BqHKtQg-e#9^G_7gtz;Plp~F%<%R-b_Za~Mb_J&}lAfH5`gG@a z0N^FQoqdh1($lca(8?ktq}P{Y$`FwLhlQDuRYXDH{ZZjiENK=U6=$K4q~KplhkRVE z0LQ3u<*%zu7GN4QE;|QDHJ5gVj+Ub#|A+c(`K;qJ;N0tv==S8vlQ9L8F)D+sPf$Ov zsdMb7)qwd@i_h(uMBcTxrbzF69YUVI*=5V7g8_ z3@fbYiwYi%m5w%d8n|Zn{>Wy)!&U5XW8vl1ZPr)k-lhx!pt{BgSgJzW5y9 zr)2kln(B!sFWs~9X!H(3Wjn!m4a#X#NjVTO_Q6~{!`@iBsSVIA_O@($Q91(xd7$$| zW!~R+Q&c2|E-S@r7W+rax~iJ9CIdx!f*K}O118SuJ`DeRK&ryW<>~z zQ%pO=qOJRg=zgZQh=Q;_T!YdAEPd9{JuzPiw``IgNESg>sDnY|>Pl;8{D&p{_1^D1 zKYXh{Q>Y{L=jaJ0t$ue&I4_U)S>3P$%D(wDYS#E{_P=N@0XgL9ZWci6o=eYdKnzAp zz_%MkGZSlBU#bth^j2|4XO!ZO49yO7Rxe?#72o@wbmj?ak=z#jA$Ws zWl2AJHva&uLbwEE_Nnzz=OFJz?@Qj}owfQ;J$tk5IRO+OmNK-%#r*L{$+GWGU*=P! z*CZ1@;M(4LgUM$}DN?glI0RisIXGm+jJ5?+Dbk%&uC6{6thQ|FF!ALfn$X|o|9RZ; zTEOCAKcezo_ao5Pv<6*oIfl>Qb*}4;Nbxr>9zRCyW?!eFK7ixI+hQ#=%HXzoDiuL& zwV7RV1USzA#nQg)Sl>Z~F`s8QniLY6N2A)-I$2zhoe*OYIM3sLX6p_T<7B60kmjNH zl$p_!hbpG>Vgcshs2~lXk=wj2N^Wc&6gP20RsSJIe?u=hsTIj2vf%9Z=oInAY=B|) zpk=k(NtTL;eR1_)Cq~%UYFE1IW^95%P!s|e=Ag;?a8(dYR0%gO6Ny-C(0!JWtAtN( zztt5zl--Ns^RGWk4-S>u(b>H(Az?JgBC~`ONS^Y+ogcjOmp?>~kQjb) zgypEEJCAYD8$dV72LQ+z%vH#TlTruqqfW_f~<48ui;dK>5ZM!C`0_JtAOsp zC?Se9KslZN0I4~{{zxx&1MfFXgau%6CwQT5JOJC6PQFl z!&X$g(9mMTeB|S%TS9!YnpK+#@p};0=nKmwz&%MNQMjRbQe+VIC!xtMd@6PFa$k2( z)bK6xYYyaZro3NWwX!ppS14cj}(5)3wN48(dHEA`@9UXSD;Jj#8Y z@%B^(!(rnOh)$1ad|Bg`+Cr(%rAdD7MxtN6^^|M)A9Ib;79EbB5;eX3m{Cbh%%>C= zfBpH%V~Op){(cFs-a&PJK)F#eC)mQwopV1@F#iphmIuIZK4Gi`1MCY!E4v_9P0h#F zwF{4|+<+VHJ3x1{AX9_J1#mr~Xy`er#g)AD71~mkBbUqaRGGW7}?3 z_lTnoPMP|ODX@kPNr(^DQ@NG#JIcd>5bj!d6@2^P%1LlnI@QBsA;d1`xQaJ>C^JKn zn@)cg3jE3c?up2Wg}7=knwD9=g5J{5&pabiN3)3hXF)`Stn`^i2>AIIVDuC9d zmSSJ3QAP}sP=7Y#F0&7S=&id0ezfVz--+H2?~hI%jjUqfTE>W*a0VjMVPq?aWv$Hm z&x{zHYMl$j+bfQ9ycYT_E!2c&a^rMCZZo}wm zUU(zLX6sB1a--T0n(xM-QYxS%&5}$hstHwk--y~x-Smn`+ub zIz`3g+x2R3L0YEiqB!r%tB03Bb@k2o;HYW_R>Sr^EH?k2{X&E&C-N~{I%4`mIU- z+qtOO7ML8oobW|$pgT3pgLpc1Vh47jkZsyc`C-GaU+hPQp;#HDR5SB7in#Cxi` zBi|{$RqusN8nJOf(>VdvCB-Pc32<0ov6}E=Gz)_Q$jRR>8mi{) zGg+WpLKTt1u#S6ALcLJx<9+dk>||=Zwr74%59@Jg{b{_r0Y}XdnDONWmm23K*e;{mytkO&{D#A!Fc;R0-(I+3Yt(7X#BTOJ z`I_g1E7ExU=V;uUUUdP&piQ}E00d`jA1)lGJ)4EMdqIih1+=ye4$cR$DCl4RU;_c@ zljxY3sGK%YX8nf8z;vH+$A&+V!!(>a-^BEW#%Psck2VMR7HGbWS594(6pQlr_m5xQ z<{)X;$NJe|kcE7_Zit@_c$+qo>guv*ay?`~%RO$pGo>%7*RQ1+yiAn^sP9= zpclsr&t}SBe|A|^~R7?lt?lmyVc z7T^l2$NvWkG5CFFD^^D+D|Izw|nu#3ZC521!xz zYX@QO?$U1>N3DLMZJPox1jfOXPLn&|tmn;#?_7DyMMRGIMizoEznR;72zLpWL~=|{ z*I#@x*5G14OgH6Dxn0y_q1-?Vm4_H<%Rq}^Wv3Y-;3D&9(RX$8G04_mqhYnPE#kr)xsox%f%HCC`eW#^+`gP+!c##-bydB9|6G;^l!k zn6lIWYb(1xFvDM7>)`ceF2MZe0=5TqCJxgNMo#~dX8EjC<>ZUM<{Q4E*u~+JC zI-u(f=~_kO%{_GeEyuQqMf9;NW|j2MCZVojo$i0I1e*6-yhlRqZ?_;TG*p|GCyz^O z7m))IG6VG%k!pZJo)riGX+<}2JJ1!IFX)+Wo(_p)EC9u+^HLZOr%f#gbK=}v*m!t( zE|YoyqSTJj9WHcsaaovk+v|*`lK-*_jc1if&^gI^^r+NSR7w5GEn3=iODz{f_J-x5 z!>rExuM|dzz*raRs~~57k1OYt0$t$WmS@4b_xi(MYK2~pz_&AV?u=W;SV_epre;hQ zI9qBGkPddH&;sNYb0s=eVt@F}8~Y|izfJw#8!JBM$feiSX%%>d#y!%Sagug*-{b36 zniJsr%l6cGvPas7Xr>^%e-60z2A~%>4TUm*PGkzR#VHU7{aFX5s?ok(=zk(sY49nx zxa6JOPJ%zcwamyw zB~vZtnuPeBc2SW&(Gk0S4fb736?9>TPQ3{70m_Mw%M&RC zk|JC-Mn2)N5Be@z4nb~2<;QF{@)F*{7C5s2v-jMQhbpTCSXkabnDPn$KkGB!YR-r9 zy6ryHeZ?Ja`toxga*;SjyB4ozXKd{7W0oG3GQ=Jeb5Idu#mYFMJIojVC&({A8I+jd z4112$9Yx@&+o)A|f>h+8613NQ^p##N=+qdNUCqcLXED#i>6}}|!Xoo)m%`}N(J>h- zw5P*4qlk~KDCfQ4u71ClNMZ5pimIUhvff+f$bq}JB(8)i(J@4WCG?iDV>tO=0!$y` zWBw6pGVc4g z)N)QWpAT1Uq?3U8ZW9by3JeMHasICMzGwCu$~WuAX(P>uNM(l$`({0i`3{C5N_L^Y ze})tJ9g8|ORj^(t9bN^ktN)BnOe|c1K}AL`TF0DnAzu#aPW8nDipWyM?*av_3=6M& z2x!xNuyuq>N>(mJ@>iH=a5LdR&d}`~URnXs5EYS<`7dTgjQ*GN zOA1g|a=svwX?#1%#+#C#ECtcGi;fsPbWzmLHG*R3-c?kjbU0rPK6vBnwO>v1I|+M9 zEtw{#I6sGyG!!LCjgMT3p5}=K6zy$FNv5B3rC%|Nx#^jXJSV*0){ziAta1mo&)Ff# z%uBzEsixyJUPK}Dnb9APy`HM)LaaN~j@d{(iDuc;m3OZF>oU<>AMZ>NOpu;~e^87o zI`pFV)1L6SLTB>_g0wLIWKl?i61w5K zyz7<|#IS0{FcjR^(o~rBe{xeio-ZPiJ}Z0*yKpKT z2zhDt!sk2u2H}`t4KmZzc8Q-h;;*74yt4Z~SP@)w)0rj>a?w6Rul!#m4L$a8Nl;iW z!YzMuZbPWTHqco*!oZY;T1f6_YP zaD-jQnGJ`gCx2c|7_q9SMnk4Dnn_J>tY@Yfnnb(N8+cVyc^X7_P$|uS>P=9!uXJl2>raD)F-*Ms+0Znka9qMQ{A|8VkO>9R?yl1%9xZ zvlOf9>{#@i^5IZ9^p_`wiD-Gn>E?#DxMH=oK_6SKrax?iXLq^&IVb9TkDAln zzw~>HM7K4wOr{iOWFakfXIuRUp>Xe12S0Sj@(Ox?Ic_FfO~RsQSNgc1DFdO*3)}JfEV0AGNc;Cr!~OX96)%jc(tpVHO@rty^+NLQHTl7nJ+33hiK_FIRV&I7Dm z`#-F9Bl=?T+9*g;#;IW?v>ea0e)CwQbh^-gDe51tkqvsaUeF;nhi^4M-hkJMjG|l| z*)oX}NOVC7tGd1^p5t$(-!w)+>Dfr6<@Vl;JX^`9YwX88{u}8<1`}k5BchZc&bnN* zwJ=^(vcPv+7kg1)WA1?trxN{n+MA`$Nw0w!)%78a`Dagd0_EpZ?_YDK6i(rLLuIiW z$q1qlaV7N1TCDS+176lE>o2K^dPdTpcOpy^plZT#9*0oTtL;^Rs!u1^@3P&qy=d+e`QwWqnTg7hDY+A2v=nyBKF?TZ3D^~E#QNL-eCZBqQZj8kX_xSq z5`mZ!GcCr~l%jHdDZ?8ujME|4>}&E~ zj-unrIK*}5tfnx0Y+GztY;p}d!eqC{t6fBpfz}VniJFt?{$rZc4XTK0D{7kLkJGWx zbiVjGseafhg}&27tjPVgZt8H+Ux)B-)ce|9BZN^Vr>AF_2sQ12ZQQ1CU-ugM@uZ}_ zt?_MF)D7u73YC_?P4znQNC|?r$Dc62Fg$1mW2*{V;Hm3PVVnUmbWL8s1xa7dK`h@& z{v&q5B&oX`2SfyVih0(yR{+f3ZmLE=R#Du*-`geea*YV}_l4H1tL^+N(PtCw@UQ9D4^zLF`FaV-=!EsPzMl z%W^WA^hmj-IzA8EcSrOpJKM^z++~&STk*D(u#%e``xUY*@#mGP4eO*oIUCw2o|$g~ zGf(i=p&u_8m;kItCe}~rQCqKDKfH)FPGwP!#vk8sV-a)J!?@voX6gT!OOr+cT>{cZ zC#U%JkAmmoNU%~=#Rf}C)n~_}XJ*C1nulTTDS;F7ZEzhFR3gHq5{#fB5g_ElU+L7Jl6Y#~t8I5mlY`Pu*A#Y%@}ucGQWfY5qn68x3})EJu7-H(!N*^ zeYV}km0p>7ckBtb8-@Bkp=jo;AEuuN*G%|sr!uEcMQEx~aAc)Ua~0oafH6hVM7!R6sD=G8n}1`TfLG^%Yg$Apb*};-0C=|9S|C_dkF2TFSYdz zzQ#ZlC`hw(?9dr7+BrTNo_NoEpRGZ0p`U9%rzh^k>20nC$nN?=0h0f-`QN#I7mZu* znwWdu$+j#Y;^yYh?{iQ@p8Qtwx3_=kB1723j^=nVV=*b- zB6KqAma|*=G^VsHIbK+^E~MY?C85s zxsO*nk`KG73&qV)j1OlSyTDirbEy;rsXnS*n*8murYmy&RO5>4fPlByv5MqZ*O-^H ztGHRLZW*9Eq!1~Bq2cs$A&rW_GDu_L$xr3c6%#aYP}N`;f4CW)5KiSs|9HY&H6Y~k zbKXzZ_Vk3?vf!r+lJaARS-1b=>MMY%+TOP>cz^>)cOxMkN_U4Kog$4vcXxw`gn)>& zsC0KrBi)_S-Cf^0UiJR|-wZR141@dZz1Ldr`@BzdB*ch1VNHl^vwq1hJ-&aeu{b?l zaGpHxlC$+~zXt3kFR`NtR3gJk-V8G}R!p*$o%Nb>E?T)FhD>a?{OO4);W}s*qMFmv z-iJ)v4ePgo);dQn-DGGfk1^R7%py(C57i+~wRJ-stG+bKF#tkJ9f34&*Vo?SplQf{ z{G8X*)$wz;zzy3xU3OZ}_aD56@_`aAB6nSl_%bwp<}^_`b%+>|spAHTspwDfd*8)3>75;E&Yqvpbot4eajMfVN*W%$N_5*X9#Q`-54bD#) z!D=y3BDD6WqO-RW|BFo7dcvbK-TyT3l49m zU$*qB1p(nTfO7UAty@nTwiD`5lKIZdrGObLK0gSFuB=OsB(})lKv*ZkHC@3{97+F>?t%SwllM%cohsCFE56W7A)(|LHG6z?#-^QSb=9NoPBZhQ`u z3;WG!W>yFcnRb^QHf5xNU>B8|d!wC5q5gEucCQLTqm^J#ItEp?TW@in5A;QBH=~qf>nEBhi>+ z$t@a2Gf4ESaY4)L%RZQb;aY}blx&N<#wlKeps;;5(aYqtOX!QpmZN+T4l#)c>VW+Y zY(5eTNs=Zk`^IGR7%xe2tl8xWy`UStmaekz&t`bh@*nYR2f;?!-?5;b zYw2X;&V%~;T&YT4aqoWY;e{lpv%Q5M@`kKMX_cbN!iOxx~`t2PYP@aDIAnfz6_{ z_(QoxL;Ln=a+H*x7n(W{DIKOof-~G?cqEVb$2fa%p zzwY5PDCh_Hsj;n9u2?>4Yo}SpY1C1Dri4#-#SJrDFAS7GHtG?FAC&@3qDq>*Gz}=< zu;FpV12BkYq_)U2~k)8CDG znjErhS8^*tDHe}OUGM&x{$jML;EEo^z+Eb??;XKk+p;Wvuq<*TxC;N3ni)z{YPn&3 z0N0~@j!^;#=bwA-my+X&I3690ibo(i?`l%vJMZR*^Nbao`wGTD7-cCJGpa+AA6L-a z6*m!{kh>wCko%D9SnCHkeD) zi#N37L5vWSZ->l1bEOjXKR#(OSR9#+sJr1fODqJ%Q?38p-8Y-~qT0SfNux~e$9b~9 zGSJyXuWsc0E=jejHs9nW5wBKmcFxG&pi3j7Lk5Gd?5x~fP6A>mNa+cf#f z>`L_A&Pw<0Tg~aLkX?WDFMmMR@mQ76G4HBrGRijti}%rM#)w9)L1P{4A89S0^7nIa ztc_m(LT&nZ{5~gHk|qk`Kj|izsEaj7L^tK%@dEr-vr>#dHx&>A-hyW|UX!Z(%bB-I z&`*%|ewgUT?22#}63~2+grTDJxy@pma~5sAl%uc9lccwt#JAqvT@!>g$vA%Ec#J&K zK{j&FZbV{!81JPwcXK5_qG+1+csB9Ojv2gLWLw_0+%yZakg$?cP8%_c$fTMxG^rLl z8{Q#xJ_2x(5Q|dEOBwK8k0>vbPQ7+0#|og38p*N)MM1^?uJr=gTHl&gEBIv;Z;lJ+ zhRW^Gp-GEt%}Y&aE!niO?C@#<3pZT!;Gw#BdLa3U%`ldAN($^cvnC;c*lGvr4&H-4 zToX*W@PfN-i%`PjP&|GOte$iv>GIg-BcU*oAXe~5hq zDgTm+_RnV@Sr$n~36>ogc5mTm`B1ph_a8$Xt1zwg_17NL^yfE06z`6cSB11=M_RuA|4X*Hd ztp#vJL&pKkq>Kg;RDF+ix)HK3WJ=ng|1EZQ75>D-3>pUSpJNnUNAstO7uCQC4G)*B zw4P;Vl{XrINBZ@a1M_9sGs3B{~@x+k0Xr&jo4E zJatn{{v~0;I!}5(VXsR=5ZH)cn!#@B7x1ee1H^98GC&izlS&|3naH(jz-fCphmEHGlPgwen-SA37gANu+YsND<=>8vTK@cHsl{s$aQ$w&?I}UC-HX zZxU*LAf9b2s3^+>fok8ci%K~*{%zI9QgoGVMDGLIL&>EdrRAm%xdn>HAUpzy17Q^tqGG6yB`SOvhV#mO(92v>!_qZK#040fVde?Zy)t@EOU*87;1a>=Dm zwp)pCX35;i^TF~bX0Alhorasm=`sX#L3a@#9AGiZN^Pp*F_7*{zcD@T;tc%=s_S3p z#0^AsykO+E;d%d;xlom{hhwk`AWaqeVrMeFE>;GHS|%<9%}!vT8Kj`}M=2+I{e??t zlpR13o>zUA!u0PTswl=cF_8pDBdFNF3Q4cZf=<>}g4G!$ueInw1KD`CLa$E#CV^2m z+mddz4molG6Jjui+9Xsc*14k9#3VS>mr$ITiv=TZOnFuiBT3JgoTu?!yru0FawhDZ z(}DC4uxmXR9{)uySCpZ=t>38OL(+F-03>tC-(0-DAYV_UCm-+7BlGQFGd z4o)aX}ieHiVZ4-mSv~R$Y`q zA67`CY&ut?kntlOK70RXx&f}Vi^?#bo!4}L^8EZfmJ5CQ_r?1{IN|m8tv#`bUw=)1 z;tZ&)U#Y0rw(=%k?9C!iVheu^(&fP*J;i8(yHh5GLM6iD_ny6L>+1t}R}jVy8F5gz zYQz?Pw8t_g_pBp>aP~Oyqs`X=UpE2K+z20bahUK24kvT|hd-n0CR%>IIdM{yZf%*o zr&Fc$J+d66L%rQ3gBk5qx?e@3Wa)k}O8%{WmC5011DjI4FSG0HCjO|MX$%8ISul7U zPtMd630TqhC1c}nE+AfEsDu1Ru@#Fnm@&^oML3SitsI=;4zpAy(sqegZAKqkqaL&$ zVcqVRxeqL9sfS*fxER`t<2NYwI=n-0z=OTc3p9|&ce9Wme3p9aET#&9PGIGQVbk*_ zhf}kO1jT9`z8R=GFSmRPrYve$GE>AZX@8_p;7;C$#xmRox@gSm>T1Ny!9S)3FlzCe zC&>GgmG^`_DqF6;vsSIGP6(-+P+)XA+prkmdbmHRd}gs+u@N{^k*uPfTQIv+l;MCS zJKKnA^mfkQqvGpXc4&|Z^*#2(U4ba?;aOi4hi+9a$hoxlsBOWzhWGg*5*P%w{5|`;TbF% zI->2URRCx5T>XTIUEB>S{f-pX#l~hyxa{vq6a~nxFHr)3(O88Yaqr(sAWkfeSA>En zk%!W0miSBl^ZX#pau?l7f6sG9I~8uvdPoDy6lK}C6SR605e6Z=M~!aNfX;*`J-_<- zUE+-LY~5(~GrZ3Y+#q?|6v-0UF>AnFA?+KMcF7Cm53G9T#oKRLYqEe+1K6A`O}JHqS)^R;Y|yR1ap<^AaUOPLwsV3~ZM#H4?!)KZMxPkphq!mSiT z>i=F|*&gLTRZ*0v6zq9vz7S3MS8M_vzo7^m4;$N#eEn&g_i=ZpQn<|Ko<;QOwE35t z3l5ZF)g5wXu%ESdbK5)=v=|lZj2ox=GJXPW*p^dfE+NTiaXtn!mV6PtcL9+YqV4S> zPDfAmYJuZ-JzVb^Yk(lT@dLj@XlrY$Az+W-&%)F=Q{@)VCz6V>iHNcjE20&~Hw20C znY9fcxq~bmOl)l9-%pK4f=6X<9We`_iTna+uv>4)=2s@-E!~3}NiDKi)vIQHQjLRV z;En_6qTE+Se~$#Pnj;I?1Di-lu~C) zxUe00MtY-=_l>$aR@VA+a-Q4ig2;vAf+!5c8jEQ8%kpl)!Rag4g|^FanNinm9W{fc zXS=Pt0)=8nF7Z@p5(IkS;tUl$1pnL0^)r(;#ex+dG zIJE@mStW#Coly_)oivjId|j~(|)&MJ$-Nd$DuXL7~Yzk9J=jUTp=a&2oWJO=79~ig^^60 zmXX-uliFA6NvR=a^BP&I#6b*10Pi=@qLn@8eZ)^E5jEKoj>9^7;FMO<)tg^gR{$CS=clI zr@#!5x);YSjFiI<|1&P~Gu2yE_c3G{s}og>U+7p8i(h!}7S_=Wb*B)%$-$>UisflSAS07zO2A}o#eMs= zU<0BZE+SJ1wal-E7@QvFvIF73x8B0@&atp-XevTz`2Fc~k<>(u($0rIg209j+ku7j+`+n8S6O?ez> zBY~*RVj6@VMa9N`Ngf6^cpx1G|0j`FrL|tmkrk1Nu`w`uO*|iigvB9U05QZ;YZA{&WR_E@rpwuh#PPb8@>xUr zF*6U1B=z6k?IotIT6$*~lL=+Szm`p~liC%s++P<+gcq&c9Pw&z&igb*J>`{{Uadt+ zV7k^k6zjn=(n#MxoVdzW;s>%qvprw-Mzg?;CucP5XvN7T_>xfX_i+(Vi!B$8!fM{ z>V%u8<1=t@{E&V%j&CeM*mYk#u!qvr=@dqvJ9f)Xvv+j+1h;>+a+ zpnFWfjD3-<{!fLR=-S&s*F(~CZgcZo)SbTvGL0dRAMmEeubi<*y*%Csz+=+XPFk~8 zG5K2v!X$v5HNNP{SSPK7WjC8zj^{50h#Lg)ni$oy5-TIUEHrF5GYjv~I`lmdrwdI( zgwAwB&N1etcZ&(H`d$|@K3LdYi~LrKPo=w3^O%tebFcbJ2L@}vs2vju3!_>FZ7`yI z0X_Y_S1(^K=2y;%gCmcn*-XdaY29%vwGF6OR>J)4cR`ZjV5N;g&&NkC?)0@=K!ZA5 zU=~SF9{`j;kyXp@o*;m&N4{iuG4?DVMki zPTDYzAv<-N8!{tC)2z#J2?L7Umx%RDgQ4aYHC)bs4vD{5{gJe9^Mad1x8Lt^{r3Z? ztBkZob(Bue6(o6#LbTKcKf`(cb{~pd?opEPoQG<+H@-N-gg`r@m+@e5&nGsmE0_U_`1;9jEMt_^xE($Aw-ciIkNef^X-q$ zfNRk8=}c(<+KC+C!L*-VY@ZPfmfy3AWM`+sfU=`D9Z7T&WF8mc&m4A(DZh{$7&rSG zdm%I8s5?I6w~)=Z1v|>q0Ziuu+hbm33{zYU7~jTcdGuqtj=n4M=$qX13lZF&7|j%L z)}6<*D}Crol|Fmk_q<~tu4Qn0N6HqUs(KmhTnJZ4XKmRO4MXLF3sVrCO6zy8Q```N zTwu7B6ZybQ50;*gZ?)Q!;0hS+Kv(0k;m5k8u{0C) zeW}b zQbwZV&h%b@mg|?kf15DeTmt=wMh*mzSbF-@IJCEy)2e%mgQiCt=Ab>WmRJ||Jf4+RB^y3|0=($U#S)$F+ zSeBECNZ{-QLBqSmNeBKb^~Z2nUe8%{{O)0#$TV%dcsuy^=*ZS}B%o7({4GEcDost= zoR9%$?_Nnl7}SKq(~yW9mY^*?UD1v~-w{rf;ntJT7k>kE6*)mDPCQ5UKt@>cK??-7 z?MCeY<;B;`o z_-T9F(hB{3;=ml$IG&Q!(IFssrt1rJljeX7*x|#+VX{PcctH;pKA9#$qsP2@JKmds z>If)k7lER^Vz<2}07v%H@)3&SGq;+Y)Q-+{CEoKMJ|S>!Zc)cj^5yAJAdQV)MX(Ial1=LMfJOJLMtj|_3Ff#!=mLr9fYK)aHYoycHx zd10p#4O5L7DlT-hHF6e$~ucs;Y?@_0{4)x|74hlJ!#81LH5>zUfSSftgfWMF4+D5HCmu zY@-9gk-T}CKtfab6NtZi6$k}gi=>kAI_N0%MUao?2Nl%yJrAisLYPP zH%|0>?aoS-Aj`dSCAv*%f$C$4>sn{7F#|^~l(;qew_cENapNhMZw{w&ii*<4pI7=L z1M$ky3C!zj=u%waaUMSPg}(e&V$$JbGz#8=mya#3ii+5KQ3sn%HIPwIh7PY2JfJE3 zPADCb6f%yEWg0e1=7xrptkuid3hNvvuw0|TG_iXy$!ryn8vjzhI9b3Ajx^@ghi0wL zX9$kof`f+xHPppQfnkjApO>?GHI57}7gXPY8}WUh`(oSi05PFMCR=uGY6Ih1sU)Sx zfgDH#x@kT|j7w8Y_q*7sbZ!AqPS4EDOr~#71h5C?=~jTdO0ZmS0{(g=0FEhXXug!$ zJVP2YlC-JWco+y&cd&aC?(3@B(I}$E!-PD%o!R`)ufEvV6*=YOQShbSBh<~vH;*uX z(DHi5hBbiD$+^pZ!*;sCUmk4#)){N@q6y7u8$)b$`|!YoSV3d=_h|})|6+Y#6kZ82 z-NSn1fo12)BaXeKoT5wRFGt|RX63>9_@7KF5~}nkBsARCZ{R5!hl<^lt07_PS#23U zg;FhgiH`RwwSMmM5O6k+S?q~x%CdmAk`|Hz1k=QCnx{&YC_@)sD@E6Nt~M`#l1*Nz zl>E%HotbR8$1mI>MPjgKw#K`I%tiE&`E+ld0)R1Ny*f9b06Pb}R=GkQHErfkkO74V z%nMs?jdA@B;|;uvO(}pu&9CKV3&!`9)eiac%b}Th3NRAYh$wu*G!GM7QwQbZkriH} z=1!Te|E*Owx`@@5=hS~6d%+W{NS-j3?K?2XShGO?pAA0g@w8JG7wRwLPPS(Z^A1@vs6_#IerwVDgSxGX-dKzbiBgF^ z(v0qTH>AC5o_Q+?Wu`Brs3|Y@`X0t7=Nfja>0^zxXO__*Ps zYIU4xtIB4+0aO@FfK)G}a4i7L_eB14bN6dOPBd+g;sgvP_x z?ODNvf=}?Tl@zQ#FM`aFIloFG{8PpIs$uUJXi-FC2uE*D@fzGtwEXe zxa|%ytEm!PhIxl%RYQix?t=9ybt8cO*(%K6+loJ}{PI5O$@vj3g2iqmiq;>T5*;>f zZJDQ1jQV}{C&l|JwLILk+Q=MmZrd|GyKx`!IDU4{ZDus?kHRq&b$nsGO=8R8tUHTr z$I(uGwV+?y_cHSkIZ$sJiACfRF(v)7D80?CnwQfYiXC5Pa4Wj4LJj4#no_S+JSbx- zeDV7A67apppQVXR%hryp1wpLZ9$23)KkZdoO~29)3h7!L$V;l@FCzJny7VQLqsF~LShcQq4^UDfsxKVWD#?HL=19E{gq-IOx{OYdn|+l7se31 znOWpl{R{va1SSdoW$LYU=a5s38no&ThP% zmQ(ctN^syu19mmLeFDLZ%bl39XT5klRbnO&`I1}5oD}Y0`{<$VQ^!+^QCTKCR5S^Z0Rs>)vUYf2-@|A^7D-v?n-q+@~B zi$4P=RO~38?2J@*WHL<~v8{ehYUBNk{jegoRJ*Z0A1r|l1k0^S{Hu`JAhmff^dD!< zs*m77ftWr@my0=YAl?G^b3QqkdP(|oRnY)ox{Om50YjJ;go~>y6A*xaZ)Im^_o?Vr zic)`FKsXn$=yJgE_xHCQ)zFHY0xC#Ew%luZ5Ct?IM8TCzna-c;bL*zP24A;RG5u7} za<_MlLEjgI!?kS{Hb#ILTvKv?;H_WbR#3jKoz_hL9A^Bz1e{q)*fEG6w|?#mTfzzT zv8l0s4=7U)mb)zeoyqCO13?pn)YF&}UFi0o<%uj=S?3GE>C2eZh4D=h8`q46Kju-e zwwD^lGLUr%aQFkN5X({muyMr4T9E&`4vhuh``wh+Rf`a1kteoXmfZ77;t&~(j>h;( z>7I`=(Wm2u^qS3Z-IG}RIT}bT1o@)tqddeZA7SJ`Rg8`>`|eY84nSfJp~WQ3y7PIc z?+EfG#}3`Ylpi{&5ed0-0;#fF0>2X@=%q7)W;l^;fX&hQLI8&CpcKIaTH3Zc0m1HF|s19S_NW z%crCICc7>uLms4&uNi1iRFW8**Sttj6BY+E&hg|}9{Ed2b$pyCRtLnFm|Isg2+qWyF7pKiq$vQ4WT z8}kn4cXE-aG%7Jw?mKz!=CB>Jzh%m6Pi4A(-)f|+Zr75Bno{3S#=wo33POzM1|7Se zeYp=s;WjV8;>K!UUMhuZSX*18SHJqvIG(LW=Lo`*?ryINd->#U>%i_zQsI%KY)Vz| zM)V9Qbz@JlY=kV>MAb}eZP`ISYZAgyOjl=| zvz7*IKO*xHR(s#X5G3Mvsc$+%Qn>}GO@<97Jm%klaRm>cmsUe%^@Xv%hll-GvIa=Q zC#d-*2S`;6C+6x>FQ3T#`gIY6IefA;A53F}*;@hGY90>4*VJV;TqPj~CGp`4D)z9n z*E9=nTrYqucuP`@Z#=89r(b;;LH6RkI>>H{$#(QPURQwyG=A!Q%^8trAV|>r7Fx9W zIjszhTyUZeHIghz^t7p-Uq$&FDc@7Kdn&k&`?Vb#(ci1Fq6jj86t}%Vc(xlCL873B ze5c_K6|jn)s_f519a;XxXv`misl_Lof7q0MjhSIJIX7J4loQtNcWBv#XTCdODS0%t zbdx}i_at4iS?|O-opFkfWDv1cMm=0gu~UZL+UT+3pi)C9qKSqz@4d*XZLiv*%)67$ zH+$92|y>4larrhQr$u}hBHk-UKO{L)KLS~T>XAC>ZfrdU{ThS z$Q%JuM1p`?dJ!aHj|4H5ue*Mu5?=xlLkGZEija%?fmkYroOtY~c&5TSY#>4Pb$vXg zw)_c1VNk!g8G8xN5B!H%AT(j~B^c=x1MNE)>4+6}nvOJ_SpPeS8-Ea0Zb5Q7GfPse z)-Fb+Nr14ag~h{5<1^7mY+XAJP$ z$IByV3ttemP-x9-5I{NL3KCSx1^uv%)#?yvX|e+ML7G?PYBb%&VV-=$WwmyNB@s>D zDu6{f;Zy=!0gx9T{5bANN?9PW#0M;+qSz#V>{sfmG)8i?oH>EAaXY8eHX)UjmAyu^ z*V#^lqZS3#?cY8!jN@W710?1N4qlrYJLs@MfYmUF%A8sR+Gnm~*Pia~;UQKUI=c8i z8HoJBs~2|Y@YTgK#Y;7)PDZ&Tb-YB2_ksmFsCa>Sq+cnO^2fM0I5tYs%`$s(RL>LF zy4KndO&&ILaQ3{S_#@Q&b);a^GsV&eiOJNNB;oumQD+V@(X{$=x;8QO1% zo$>IvIgY!wl&?;&u!fF^s~Gwp;tPS{n@NDf%xE-vkRi;k9vc{QVbDa=C>5}Z0ns4UFtFYbx zJRBg?VNJSRWYQx6Zdhv2o>UtKdOZNV;UXljudV?J@(_W%BL_OgIlGh?86L6h_iAL> zh~)q!^ya3Jt7&|M(QY#A=7yb0A5~n(-Ra)R-`F8A!4d=1Ga}oGzXS0FT?h@QX=_x+ zz^;!^`UzdG@>+qPEe4d)1q`HYBaF@M)1PFCqLWSQPqioajKja72 z3EAk;^Z?O7Fx9~lMqZ*e9b7yerIDgmzRt_yzNP@A=OfSTEa8z+U*+2VaKn!|hTfMx z_rmxW2({IRZ(~$9{W*kDycBFLdtn#U8+2Y_n&CBPw%<(SdX6-)-SHo{9%ilp$kaFJ z_P2k*^uHrHQdpmIwt`@=2sqZI|q5 zr}_5Y0r$zseTzxWM(?%3Bb?5uS{#qPYIGPRFO{$njB0;$03#5ZUahb}>(X-nZ=D?n zWlxA;npG(Hd(eo=e4p*7D7UGVnYNE4mmZ3zpuWgm=~Ut^*s`z&vvP@_Ki#JAfvF1& z*A3JkN5ZZt`-We*FI#qb?m?>jpz1OA&DR#&pR?J(eDO^%n^I6>*J7tQe1W1NnBuMj zE%(M_31D%vwe=j#(oAqnlZ~I7OwRuT4(a;lb z0&361TeQA4Wr~^nFEa=b^8iSnmOHCb@GGb)1f zfg)tSA9!(z#HAxyzS>*^1Ps7T?S+Y*FnkcBjQP)>XO1|9K&S}HMVVh{8vty9fao1q zxzYgQPmDQLzw0LnPY4CxW<~$Y-K1H_fe^}#XzVe=^q=6&0$+xJ(^M-U?yegEb6i}q zsZ}u$B7A&5+KfwFtcw8*Ns$B!YoS+&XKfpC^RY~fI%td78ZLPDxf{DYM=e*zi)Az0$7qSaaLUa{zOsgJ4YTq6{2up1(UbTR{vPq>CTx% zbL0oDp}R{BAy49Pt9pJ)>s62_AfP$Nz&gsBE>2P}j2ew0^s=fTyzy?JRqe-O+b@(Y zLK5Z?`37}u{hRvLm*N%`bn0&(|J3I#e06jogxW_zv;{BNPf#~!ee*(swvA@ zEh%{s8~dNY%~%r^ZLZ(jb)hxR43~&T!Ln4JQTJZYqBD@3F7ydle)sBk3j&ire}YkJ z74DE1F8^Xcju2qdh2=f3-21j@bcjNE>q9LiMqC$jTbu0F4jrlIvc6^GC47>r4I+vn zUA$XZE@vcfcHWx4<&{qPd_kzkM}g zv?kG!VDP zI{u&iP#*-o)=?scXEjGRuvYo5`e5H|sBY%b(bAHwXS=Iq?dST_0Y;1Q3j2P$Q}lA{ z8nWJ1&-*-aJ_9w&BFW|{g}*Zs&*zG;^Y8!i8~`h`!^~bjI{%elY3(>bPKxxRTS*xChlbN( zOi(a6{c76dO3Uv?d2V}CU->w${+2gjNK3zx5pdc zcrq^Pl&~f)jNfu=%8Ek4MDqd9DZOM}bVx%Gqk+1ebpiIsk;ZNxJh+?px}HNA9obNk z)ni4^?@>nqhP^EL2=ER8Be>Q#d}>Vmr2P(#@N)G2Q#0!1PqGx6^=otY7j_%GK@%4I zI)7?EA`hd`eK03+v-_tl1Z8ACU}?A`xBoe6*Y(Zbd^##@{0q0kW?RK&EB<#^fDzh( zvI9gF=>GlaNnyTPJ2j!C!q{NlNKOHQRGBB7s<-|#D%}a~H&F79*9yuQ>GP5k@4T>t z4FkJ{zN4%U5ar|FrPgo#6Z5&VIWo!N{g+S(4q_`u64mB|Un_pq6S$xKg9VWz^9$-1QQ5ST3kVAN z=aise+R6nFD{as(grUu7@x*?3GrC2Z@~wee#H%?wmZmED@ElH4*?hLTPoKz#&y)~s zwWXgBKp=2pqL`Tfep80Z(yO>({qK)DvEW<@IVU`PrZ#Ugv%0JHvf9knumaUpAZ&4M zCep)&GCMMj)en`3`c zvi^O+zrUShVYa1F;5_-?%cgt{DRNO6&QSVh)!`|g%$xVB<(_+P{kzYM(i6RtXLI5r zyVFu`%z2;A9n??l?F!xBYL0AdjAxH=>s{Z>7WsYCbw*@az|0d*M1+^3s8eVpolE|3 zw>PX{^8K+Kc=Yt+(C=ziT|t?HWw~q%^@`*1TgABUYhR&L8 zXUkY2&Pq<#U1N>z)I3kekiVq+bHl;)i;7WEdfdlfHi=gJbx(dh3{ffZw7CKg?Ax)v zgeRyeL{Cr+T;Goou%MGsiHEWN6mr_vy4$-_eXG{5(J=sz;GLnf^ljwgNAa;h2!`IH zTX`FsXZt)}`)#BJRzy}~cY=FD)j7&gX3q_xPtJHEOYh%z+C8|oJQ+*G0eHF7W{y__ zqeiY`T#ovRNQFgSj}K{smMrY$_S1%nHnsZnA*Oi9aTod9kj3Ralb~gaAHMu5kL+<* zkih>&PO}gtAANqO+~KuD=e~-1wfnmPPkU+s$Oo?rgHLwDD`lINrU6;fNEBjLY>m-5 zT=UGe!Czwkd+A`$P+Sx@;E?KnAF8q(RNZDWSjWHTT7xl_WaA0j?S^S zA~(MKAyLm*6!}7k$?R!RX>(;hFtjFWY>sB4-B6(a@d8k!Huxgaw;|Cjd^IrM_RIzS z8yeFGm93+~Fz3gZ%a{OqryDvo18JYd#C%5U*vgQ0o(Rd>>IUMEnm)tpx z9%r4N_G*9jhhN(SB{@8{94EQrdtln}cX|1ZU}ouNJmeBOl$`p|V(8?3iDvG&uqdgK zBRPh_beSE{m!5h$AH6Cp7+X&oMs-B+v$w0js*yfr?z`4jJ zL^(*z@TBYq=gJTVyb!`7*~DV{n(5Zyo^p&28Jk=a%F)P6N`KJ{@ic6ofGy|QVAV9yhMhq!h!)e++*x z{_o3+2x&E8zb9&E4VX^Rz^2#a5*W{$HU}baR+*17nh$;sx|{}dm5os9h_w2%xx|;= zgW`=(cUYcRgahL4?hofKhD>F^{17r**`YGk1s1bkQQ7L)mz0HXj@8@`f6#3JVCoAI z$EyM01C<3b-ABI46P%0#U&JZ1%;d*oG0wqGq=>t{JoD-|LwoJZKVq_h`&ZfIt3(xm z*Q}=npnq<`5J}Pje)-Nv*`r#zx=%5Lj-xUgz<^@_Tc>ox&%{cf?$u!9c|BxcuEICCS8N_lv`oGQf4KxjCH{ zy6Ie@T0hgpTFusFk1p{S#f5irnM~fVbe^-+*=G(R^4dMbP z*K+WTIcKm!PaOxr31co|XU|QSfsdf^vaZku)zv5cJzlrjTavw%qUh=ag@!KL^bsuI z_b!Y~2=bZ*`{jB*g*VwZQ8LJLU&McH-MA?5`@8d-b95s-ZkK5B7vaRO=6y)GCNk+p z=d3%#H|(o7Wd^Ip&DvSChNhkkw_NoueoiE{`y5u{TQv0VpZL4{HmH=}5NxGCoCJnL z-#|k*44js;z@yoy(1kM(P`&|(%vZn!j7WoYNj|r({;Il3X8Rp3p(J>Y0yuEVJb)#; zf8uzJE~cWEf{aoR9?lF0B-{!_0;g~u!OKm6ar6Li&wu^!sxKYNw^w(x-cDlkh2LrO ze0Pm{5d;!wdciF7+cyVc-h=~Il?U*p|D-Mf`)>(ACpikw0H$x8U|^x7V`dP$e=E6; zG)JZofQa@P_=`O=z`uU9ULRuY(DQL!ng0o5MZfSBeM?Z*;{&!!@SxR>SnVK!Ze_w(YafWAJ(nL{v-geEC;NkcC=l7#{W=4frqCvdA+WW@+WgQA%FL2h4Guj8YU4e|>F>P~&9oIAcC-V*Vf7?h8G@ML8lW*{N zcQ#QUm{}_My(^$jc~RXN_>W-Y0-P(D>a?k%d={ox1-TDWz|>vo+or^x$mxVhipRbr zv*gS7nK5ovfXTwS1N_*D;`~jC{c8cHHw3^sB?Hd`DRTFve0$K~mjRxqGU?Z}T?)ZN z_LLx00(YjETxAI0Jg%vrP*K+nb>TGSPJBz_6i}KP2Nuy!%HyW997WQ-prwfN#m)r*nJ1Au3d3zFC%z_k^bF92~lFKH2QQ>QA zmfyvj9=6^5cw^*=r|n1cfe+plV{K;dOt!FC%43B=+f;7;i?3DXtaDcpTDw*C{@Ii1 zxa$=XxICnTzJc7jUCm5lGKOq3)TQdNO217 z2OE?|e4+8~rtqpkOiV@dMZ?(&!;to~McN%}VrZx_``fkfB9DO*Z@}t!7W=Sf{qx&( zuzSkuo&t`-LAl#zcILx^%sPVe`MX;gud7qqKnn$NwM<2gezXqHK{X|zuPJ*Vs0oJ$ zMrXyrM-xB5Bm`D$bkw*f&`p!gZk1PfKLmU_EY`D8v0mdUPWmqo?8L*P{BOwrCn zwB2YdZ<=|{9S0gim*8Z?OO3DZOb^|>%)|?=yKpc(&T;w68_tQ%y#~4bW&@iYhAwWh z0^Qis4?{!`I+jd(!?PPAY??e=HL}$%=LHO8?4SCsHQpu)xL8M&KRHl9@Iyewui8^s zYJbNQXS8OO^wsZmw*za&C$hS*hxbjJvOyE6}a=Ni0oYyWfTO?^62wLgl?VEgTkQ`8Z(1K6nHx<-?d|Ky9!H8wwP`j z%rcI_IGm>VDUbEE(FJAq_)Vu%Vqp+WhaPHwO}1+% z>f*nzX}pi=c|0hy0ix$MA7nauyB0mjTLCkY!Wd&RD>l+Vl^L-0nZ&UiKQ+9yYDc`< zp9~!MEJNmmoByn|-2ru_$GtL>Q6VlF`(1yi& zz+wqG)M-tzR%VLWw2&<{Sswb*_)<0qOLWYV{HRNOFe< z#2@{u^TVT(UyIuwjUWmVP0?8Niq*Q;T~$mJ?U9QH(i zRI}ffIU?+7bRtf~aZ@92%I#rMzU~56@veG~YNfvR>0_n&?PJs|`yGDHrFNgI%aY?8 z9<#&mG1)gobXoQ%#0^*Pm8(GIc|l~bhACdAUn%6d5%k_S6k}mt?73wzf}Z=v*L1ym zk?M2wKhC?RC%Q@+pV*FyKRT|=bK5LwWzMgVzo$fBpp)zqbcvsE!+N=Ir32an4Xaq| zW3WzT5&VWFRj|!Ey7y@;evT~SMpxC20*;fYLo7V}i zQx1-i^_^F^Ai-AIFFY_;!w&9Y2rlHp?|r;@I;?tM^VqEm=?e`M{-v+zyD_V0zRn!w z#IAfShFez`+xbP-bK3(cc?P_xuV&rIdc91cTw5HzI!T_dno!si9qt&5?OGktd0q_< zMb4!^d7YXio~v1)GTgv2ju86KsgW)acQ>SP?G4YzYsw;Hk^^hkn;dz8oiepA#+V+Q zFuv*v!n^@F>@{j}Mfn=}#J25H6!yTDKi-GKkvI~3#**N`GqXpTGgpF^pUmN4^6K;- z3Esi>{dEFWXx{mTz$dq@xU%c5erl}j!3whS>g$K*Rd-_17w=~R;Z6aQ{##fe10rD4 z%cqLjzvter(_TxIYXX@k(M1ZfVAJ2u>k2&wbO=oq|8GjE1>qf6#~TvBxf{)CcV;r) z09y{jk1o|y)y?5^8&Fi>=NS(qEm0zMf9P@VIkMb}u_wWjZN{)<)+p{X>@1X+757H( zAnx>BJV0r?D(x4}Ma+A-!rsb+_ITdusE%{$12@D^Tq?~z-8oPBD(GYDz77X+m*{f_ z)4343=TRC{z<0mLGJ4IX?jYErDKNZ9Ie2aQn>2>-d74eOfKV)_cYU3DyJZH@$lCW2k;| zd%`=AJ*H>FzlDopYVP zVB59#d7p3G_vgMxI}hRmI8ipiq$Rf_lqLNlD_%b{*!XV{%>sD343B7t|KRhbYl@GC zU{SxBz2oiQ)5@(-adcP&>&CD)q+Nk@ksy8V5`!{)n&%VQB6L46-1?hiaVuz;(Q~af z(2)(|nt~NoF?xeOT^dzZgYR4Dd6bWQ9*$%E(d*d#!My{C3+Ld~x#x8YyIOUg<;Q#K zupck&gp4^R_xZCq=9-`$9)T8q}s#C(DgNx3QQ%}C)d_|i4}#qW=teC^o7F?n5vU#lLr3Mrep zt4D*y(ipsQPe2*o*E7xc;Yr8T{<8PfJN$R`_!UZ@%lkC`4u;{D)SpBwtq@hV4z8W@DZrbZ7M1tckUK-)eW_`?M^_#RCcw~hdp1i0? zoD%`yIuiiR=-o{3=6vaO5)dHakiO;hnX8bi-yCq7fn!KakbW5JWkekILpzzd7Nz@E z3j^*xYXffpDv~e$MVlxt^ZG;3y-tD*MllCYGLN1dI>M{wz1FmRfHO(bQ?#lF8NnAt z-5$;gdgI8a$N(+psv?wgBt}!@InOmDHpowt-g{$n;^a)UuLoTnfU!ECqMNT&JVhe- z=Zi2Z{A6?oQ!EL5c@WP?hv+u%9?*W)79zRcC|f=Pej;m5swS>HiapEo|pisWhIFn?@8tB9fGX_8Bo zmi>lZkV3GHsMD%~Z!ypAd}NEpQD9pbTSWHJ{q54DM!|b3RmI>VLhaBuf2rIL(PEG@ zB$d{(h{K{3Porv>++Viarq_4&F6A9)XD@gD++wHRMsLyLutIcm_3VUI z3Qz7=TI7pQIEPXtmL`KiS4%aQ)CsW{&?osn4{kFgToNvS)oU{BDE3UE8vp~bHHJX4 z8|!BY^XN;wG?m=@>o5Ttf1#sViNLTK)P$7$i^FZ;_2iP{#J98WY#GA*{d|BOe5<)5 zbwQfWg2}okkB8l)UPMBm#9Tv~c3Ug1B|EvY_RClCo-Ai)w$~rC zMi1{}3u0K|eA~Wc(wU1`e3KQI;p%`8I>lw{R+nWc{?F%oB1AcIBk*xkx0HYUWltIL zWca@Y5?}eKhtNhp@ey)FNn*)Slt5d@XG5XhpiP13S-_+;*kMmfAZtu)XI4(i(Dymo-^{UWCa+)#FTBi)m{=paX z?@r26T>qy2l%e*TN5DBnkJ?8(y$5-K)C2*{sjSIOt>BC4b|i;lI1l!)OOM_vaB`lX zw$t;dQDe;mu;w+8bEUf}sF`xZynmaq+emJmF|*0ZYlPQ=+cRDfLbPao=Xb6v z8mB4e|HNyPR0#w|Cl6-p|M_Amk|&&1d+=mEefib0l*KNlD}w|+1_l08t7iFcG?;Y` z5_j+L2isBpVKmPYNxatXDyS$Qy<*pLf6*-UF6AU_#uTT$v|S1;J5ad?wpw%-?+D$JIjP)A zW!IeUM7K=`-2M0!h`szDLj-tXRHV&*D|%@is?M4WltA%4id_{UC!t}zZ@WI|L<-`* zlz*mMt-WlNGS__%r&*6@7VU@qWv34Qx9Nu-LyUUeenZGBM@a8>G2b4HVwBiFTiV`f z?*y@@k#bfuI3zmd?K88mpgD&_zAfJI(`*XgR3-pvcSAEzo0=8Ddv5BwjPUpVu`rq#w z6VnY|M>O3YH0~G-;Mfp{VY)#`S2+8>tCF|$KR716+TP%`39EZSZ ze{?haa>!E#x%9E1;IFT!vs?dv(?X4xbr?$b^R3oFh?lLm9TnaoV zjsCOPpJL)5r_Rw`&jn(~HivJ~^BHK_U5IYbQn+8C-f_-JPAs~SuGRNQ-?h{wA3w)R5c%u~V%4-m~?kdkwz-r?X>mYq`S~ zwCl(#I*WLAkn?6c6S)%T_4EY$-nv;1aUpovd2c@xWD){5V4nnEKR^54 zMD};ys(e?1E8YbiLMU;i7)-h@d-Z8etV{KE;(jDwvm+CCfSt-P8N>iXdq9F`n&kIT zJiNb_`E#?Wxi4-dj*o$B3%`HG4|as+%n2|~1RT@*&JPrGvep(TPbayY<#}k*srlN9 zOrc(es!Ai~NdX*p)8~}#322&nG1J=1mYyoP4*7j1Mh3S$_c$a2vxAs?C_mX~=XNNl zMI7C|(pS@;-M2#XTIjCWI$j+4HGmxY$!b&h7?lp4nS|O;C!`t}==^q(O>ylpPn(^= zcr(c1`XfwF5)mBG3!b94aNz+FNm;)_(fm&CnR6=_A|tHfr^5l6!U-9Sr+j`@`e7#h zK5NzS{inp(Vu}pE?80a$bmv=4NQ@H;dKZj0U zg^vSWV%xQ?AwquLwvWd1#n>I^QId$R<(?ugqsu>W5DhTq4n?vCV{2ig5yHHdq?DxD zoKiYp=QE`+@vt3(QL&39m|Isx2&#C!54kS<+Q5^!^VPIV`lwG-DHK)2JU8}cky=?W zmhHf`R4}ReWS9%FfXzpmqHs*iV!X(YjSMC@Irz}`5_a`_jj?woTrlMphXxHPy7=kU zj*rIkTkR_nK9)g?;HoAcxC=z}(!8iUlJ!t&`Sr?|XV94Io3BZ|t_&A+gj``uG+1P! zsd7&e+HuF&kPhyd$m0ffy1p4N7&WaX^BRU6O_kJ5CuOH%Omv~muqJRU)qYKCyVhb$ zpnT@k!yNZfvWgOp*KhF|+yi?A!T&?hJv>%X-V1(xR$qji`4$5%`4_UZ4!adQ^= zy3bsbke60Pz#1J%d*Mop`Q4IvJBI1Ytl0S}y^qx$J(-W%ysp4b6X8jru*GetRaL0{ z`ePeAT2J{;hFN2}mLvkskIH3O`X}Bc32N>+{W^zy1G$x4&>>6mT}J}wU^5Y0F-X`AWl>-_DAjS*nSjsma9|D zBgWOA-)T+h&|aDH29MtpzLLx7CiTVG#V~3Bh3$AupR$YqKW?>=?o!=m5|v0XkscC6 zZ4KO@h9shzt9-7Vjd`zI$ESLX@oYDTP6ln4?wed77x>A|W~j@dj^?myegp+<&BOL^ z0Qx0Z8`V_2JjU%j?iSNZkzKo$VT9p4TP%&Cxv1xjK zGLW(N(DH9tq>blfM7~ceIJRHj7CWdDXGwBFJ3$(zF2p6CeGgTjciCXx`}2DN?nOkC zr*0!fHIa(djg}q0LdNDtzSEBYiwnLMz{Sp%ILLN6k_LTm!|Y~_m0C>-a%Sk7PX#Pd zevF(BPVRLAnH??h1bQW6R`J5`82K1Zb6V@ps|rex=NG!GaS7 z)t)aJ7wyW>#K9B-7vco3o7C5f;Sve8Y|i49M!COSd>3oX;+o3osXuNvW7AjEaM%|9 zP+(HA0y8}cL~`N;Nv)VfTL?af6uDd}XQ4Ph-|6Wq+U67DYN>D$c97sb4O7EJ0 z&A0weUF351j7KdbN>eHwrI?FZ8E5XTmZYn@u4^e1r%)}hpN99&UR)9U21STh@eFw} zcDL?bm;q7#_V0cZNTDMwns2nWB(^esR@Es%(o^i^@k*#}uN+aZaU@CT_yvx_qkaCf zvVEV=6NY&|jSK>NZhv{r$}`My&F&-Jo&~cmE>$u?^h0K@g)6h+K&0VgwK#O5iWPS_NXcZ5jIoYC=8dstbxe4!zX8n zv#~kb_LhE&zYaX3qd$8}#v|a?vd_9`!((A`kEKX6jY5qKS#vjzj&gJ|`H>;wHnlzz zr%+Ope-5nH6Qg8P;jr&C89C&(gR72d_Hv^)p#NB&V9)N|I(RhC+5e7`lLlO~ff|yl zt}-jWZT@qGA}eZgNtTGueKH)U7l#@;S1?Ya|Kzz8Y^^+BDtFb7F@DSJNvP4Xp}GVW z-TE#o9moFz4L9mJ%?x&Al#*%o&~i38cHlBUD($@F55KwC-z_F!`1D`MX**Q))pG1n zw5as6Sjim@5BdNHQLIm(zQKQJ*{HSj6FxAcg_WN2n4fLEsc+$>V~=0$Cr|D#uoEr+ z!pmuMl1z~IM5IW<(?^prG`T`*@#R&hOOam;&S^q^6^&Ep`nWR<3@Ii@#D!AB_ojY>>*$ zvS2BhQ(OCRBwfG!g{!i4?qBmVeP<(J7|o2q;l$MSB~gmv4M!XZM_X461u%1V5rr(6 z$4VFYVOmWPLw0fT%-UN1iLO`@m>c^NqYBmk6!W+NGHX1ZI~p;&|{= zwjNT3HbK@LJr#%bpS8%Rk9CcgEg_9+t9!)C+qMgG$9HJPGkTK=kD?cJ0jO|X7gML8spT647< zF|3)?(rR#eO064%%Mf@0IP{aj6I=cv_jD!P!H_!t*uK?c(M~hVd@*!@aLK4Ts2-<} zIv!o;izoa(QNzj#^cqZfv%)aOzh&mv_D97=nrh;kmu@Vb5(g(!cfh3>;YY_MVQ;~W z?3w14&r>>at=g*ndq;Gzfson&XYq+r%r&;Cr9DgwO)Z4i4ULLxB-i-wW*O*fBF+?E)P27N1qq*&@+coyw z3KMh+S}6e%u)nNGGSe3n!ZSFD==8&AT8ddQep?kba-JAr2Ju{D)8NkfRI06-zuyKu zx9^T7g|H2^K&fgRCz)QGsb9w-$~%=epP^y@1_1N~IGAcelqV^6LF6=6;=~g+Lb(Xo zxcv&i@`E92b&My{1@K)p-h?8^qwgTBnYi)_{HRu-veoH?cDyw&(UEIgF`R-XEk%8g z+Hjli5X3dTx<+leeBnSpUA`XIInPa?VQM7he7jp5OuWb_54>$^AQ~#rj=sV_Gh!%8 zvz-E%i5;u)jF6&Vf&8-;Tcd8~PpQbCvGr(EF;x8TTjGZomx}C*a`XUX9XF7O`=huE zqkrpl^Q}8gh>t3`X9Y#sa{nt#vqy6{*Y?v*FLX%kRW;$BkX%~nr|mx`F=f2BjW@5X zPt<3olCpk;WK(CjFT7~B7IBzp|I(`EZo^D=gmnl~T=qDt73lHAvjyC%Aw&bmc*2=D z1mz3Bbat2-JmfKLlzDL3%yEsyq0Hd)2CQR_r-vY?|1)=3UP5Zi^KH%vj7F&LFpBU#ZW&59Vf!|f)a2cENHtzuKROv9nWoEK|;VqNJ?#tPN>Sm(25C4w;v64&7BBt%8 z5`+qR<+nxE4aqSSR*$4OTOJ06g=VDl6ng;>s=&s@r`nOZ+unOMX5 zNzzH1U9a9{Qu$7uW6{X(`rLf>?rY=Vq4P%XX6Hpc(4E2SEdo!zv?V|fubt|y@8mh# z3a;84BhU;rc^BPYdE%CFfD%QKsQrYM&`m;q$_8zlA2^?RT z)bQN`Gb}TXNy~_|C(;ZU`Bi#!8BoAHo;zhYolD45D{PwMFd25r^d@vD)?CX|IRRZ) zUOYHMvcr58T<|ahzp;k@8(Wj(*=Azv2E}_3cWLq+xzU5;TV7(4s2Mn!;cGGFA*5Se zo5We*jckzQv`fdYf_O!5`9@r2I0BYgzLFuO*MiCTsX~j!bqUj=00&{5p8FS0MET_h zg)(v~`ubSl2+n{J-&RssMG?O?I0DQOzVe4g0kH99GS&q^hysYGUAJXSTX2`>Qa*s8 z$1@wF3~U9@E!LiF=#DI=6bbEL5(ntlTSBN_OD9c|({r{!NA zorJTTF=9X$3ekY9zIb}dl^giIk(b~}NWJW90hCzPY5p!4*;*Xq!cea3_0abhH!~H%{({ySrn%=`7 z7dZEM34%6+dvst5rI9W+1V$p>QYoxzYdDQYcV+73PIKaz48{#FL8c|Nf3|4JizHj< z{R9Rbf_U?~FJCIdEn>xZ8Bir+WoG=PTNp2bX)Z=D!i8@`T}tX#M)R@=VoU|q8|_B^ zMEi~;iMmT9j7ssZM-;$t7Y%PbaNsXg4{Z8`_%G-1X&M(0w&S+sio&MC=tTjA2!`j} z_pvdbEVo;gt3Y%(eEqO+&xp!YaV#4nodv+!pTuP&(X}zeQEkm@?oirK$y;_KTL88+ z(jw8LhZbU-M7&NgLkPC4>yh%&N^MNvhIyi1=rV*nR9HtB59*`$q8% zes_W02qinQWHB1>syqsMZe$2-{72>=@J&{&10N?bOg0>jbmE)VPrZ_t=W9CqQ4a8z z`gTeCocNVI@(_OL2f^4`#g2!jQz&;Vx*z=UMPU$|y`|9DPQ%T#bsKP#ez%56I?4FX&W#VnwuvZGWZ7UqC9d-uj3#y_G3u< zdOK)>6I~q7Fl&8l8CH+ok)LZie({G<^2Zm4MtO@jzx{4g*|L^m{bXGE^iL+aD_YvV zC$b^+{4WnCOgjj4>T9x$4Ww;Y)TQdhj^Pn_wuyUQ;OxAPkM{XoAJE;R-qD*YBbRA) zV;P~R$RBL!O4sPpkla7|)^1y2?{8ie4nx~xmB^pfLGcdqiV#1OX5nXTkETRVFi6O{ zsw%OC1C6nsQt1-^a=DZDx^R6?Sy${-sfB8ya(x!^LF=JN_#!~Zo|RVy5^y z>%R-HLuU_r3O$bg^ww~BEyW}pcRQwko2Eh9mmqdf%zsbz+vcyttp9KZJhxTxO}avfJA=d3hkhE&Gj#bYVA}tR%A>osoG-2_0(K z0<-h*T&~xi9!=78Eaq}~p-BZ7`#rWL?17x&6ze;DB~K1l6nKK!kOm-cUY>M)#toSD z=T85H5tkHpJNBZt(6F%1~IE1x?>^ zKkvidL*_dTGE{sOLb7K;V=jLXz5M$p^}woUwZjlvkt?lB1lo_72I`l_Gl#7nMRzQ6 zbg0UH>E89=eKnfsB+TKf3ZdIsib(&QvFtHu%DzC}x>HxswUNG5@T(pyc67;mk@Q^j;vMfPgI5Q$5i7I}dP&O7&vLr%OtwVQbCMfQ zPajUttV7vD!F)O%e3{OwS^QSFhEz#*q}E4C6yY>3z9&+_GTNl!Y<7{%8XQ8o0sPq! zgWsX(((Jdf>r1?^)S(``@iARPE9Lc>SM5XlMX{rS*3GTo6Q_u&U`k2NA{MP`%@ChwHr+E3pHU5AIzxm=z;VnB z9jSFSU)G!u>(qD&-n)@b)Lfh5amCxx`jG+i9K-*%{%XYvl&DCo4@`|#;3otTw<;gQ zX9tAI)$Bg)4q6z_Tt^$|Wv)uEc%B{LPLu^V(SMUR-e2#rjZx7+g8#6SkdK%qT-=dQ z4xveWwC->7<(I!FOvfjR>!^Je;E*8rR+(CeU;H>mg&stiY+&;B(9wH}+noT{t$}^r#WJv}A)Di1wL^ ziO-<#ND1|s99Dr{eODM==OC78cH7bWF2l~nVVZRQ=(-tmL1BM11QCD_=ZHviac$9( z8Fav0_pe#;(qRMW(>DDeLF);Fk4^Dx*x z(OsVw`QGMI;@5w@qn<4JSgI#zeK@IGVD5(tI6ICM;4Tp{YHskI>Zh^^CC7L+2SS$1 zsi}Th8Z)+q113kD1mDwD2br<(b5k-E;rb(9hMZHD7zzroBiSQP>b~#-~CE~)alWELEg0@?g#B(-5vy+KD4=5gZT#L6B zkSLk3J%e_;f+4|wu?s}#knKB>@x*NO!;(E|>GBM9Nu5qRatsF7Sg{>N*y^*V+7Q`4 z2+ln}(SO%PLg{`Tobmfo6op@VS<#oDWzg%Orc3ra3jdp@gUyHT`T`IABQYfpYai2` zS8Rl>f6o-*oRsBYKQNGFxpE!M@y6b$Z8Mi;GOrB1?E8wV?osF$^hx$snZ zO1|X+;B@@!m>NJH^ZUFeqMv=1F2fz}BiY~zDGD{e^h?&uXt8uZti%ddZLEz^o1>9Y zALihb&4#Bit^GkfO92Pj6@j#lo1rMkQ)Yu%awQ;5SGRm%j6fh$-54C2dK2!$zo?)` zEhwi&_dKCXa#{xw{5M^NnNgSc-w}1!O?+(!6H-WPlI069YW)H&H_ivP2^9H*&o^P# zTE6LW`o~+0qkGI^V-bv~#E#Y+=hE4U)dUx`NeZ)AHFLcy*2e7H5*v$XYF>1ssdBWk zdYUJrnD4Ks2-({`AvYpRTHADdctFY5XHh2Etzj!1QB^t9D8n)$616qW zfn9!%rz^GJ|IK9!(moy;;In%6SmYKhxi&Ww&B#O=dWNfsPhX-Jv1~DD^;Ts%epBdW zI`1o@LC`4Q$4q>cZ55P2qB$diSj%@|5OO0Zc{$|);g1ZGe zFzl6)h(b1C&u~ACC{d@i{-p2A6f1ZHaDy4RTNgQ>eH6vvKt|0U+^tYV4cSCRd=N5q zH!=)T2}-6oF~z`V;ZH?4RwTl%isY`_1ona=I#q+*e(CX%=O4}XD_j+=4d1nz&Lctt z64TfqbU9g9)V7^W2ggR9&5ngVR0SO{XHpChWJd0~Fstxux0Kl^-dhHUNIR`CC;It5 z5|3UJq0P(gPc{R#5b-O3?acy-ta^x-QpqFbhDjprkR;#^=y~xHC8fL*Kvfr+%=X5M zfW>(BG=8Qg!$E;*&g+Xgkuw|mG=xOFzHd2vd&T{*C!f}ikJ?!JN?tkRs|Xn8q&ok2 zod7$^_UGhNcx600<>tPpNY4lj@sr&L%p$6Lw3~E^-Tj<-KG?!%s(Q5RFpQUP2^UFx z#E>IQ8P)RQfwEm5S_h#2U)J;D?9>z3q zTZkSg@TC`%mGSb`HRHt>VcSAipMt+DT^*nWE6IKQYNldxsjGZ=ZEHft!GAOiVJI~+ z)a%rD7EE@sN3EED^#NU>%cG{##dtXR`r;&q!_W73Bn9b(ya*H%!^|{Kky-(4%8z~v)MAVn=wsTkiPrz2-6))tlL= zyo$dBJRbXHagmF$NCaAAuSl%KUgqjdIkJJN9fPIB5^w5K%P;jr47(nGAh|bP{&?jJ zGK3ah8)nt&=@lH^Ss6)CVQ~rm_lzGO3sQvxJ~^KM{m9;>YJg)}DZ2J6oy--I4>V*V zH#dYGVSc^23Npb;>P1kuGm;0^2tJDt=Gm@bnpn|wHPE_wjmV`km!z-wV+`H@qh+PD z6)94eF63Sct(kk5o?z9D$W?}l22?y zB2up^9z)cD!mq!>E)lt8JibA4yJquRI;=7>)K%ia(!gw;NvxFs;`_YuJ#+La_|#$; zFIduJ34cb29#~}iFlqS@GX%GoQG$kO+_`xl+2p|D*J5ZJNI*0`Dk!$65r!DLc=fKZ zyL;rY@GcVV^t%XXu-eI!mB!8T`C}5C3MWo#!;Z3VePm-krOU_zy_NKxp*PR^bQGOz z==?1mS9cANNE{?pLcoLs2NN8ro5~5Ldw6hj$B`W!Os6{>Ew^zs;K=Rn`UWG00@_#r zx2&k?m*Lq648F3Ea3E8A&@ZJ}hQ3HF%M}}pSgxbT_AnBXUu;fkHm~tW;`0>kx6PY- z$OWyww1dJ*v6S=x0)M_gztYwg_^eo>-0-z`qFnf3_ndA=u$!jv>-ZCq16m=CrkV9N zeP=Y$8Ec~(4X!HLMhM_Uo~9!Ut!&1 z57T;+@%qY2&F3AM1>Feb;u!{A@A;vzFwwHq4v)K5%HYn2Xf zEjrI7Pp`1FRmgw1RR51Everh%f81)R?_@3xhs0DSNE{utfqv6lHKpEsCuwF^$+Rl6 z76(oTBenpfM5#^a^} z>KpJH6F&8>o`_Oa0t$m;eMaPrYs20Y5cB4aAC=^a z&QD(dU?8obkYh~e*WIhA=k)Pe)_d~%ibJl>Kx$aLg+iMnNb=Y(eewdlpC>Jg{imY1 za6?kle&?$93On#5^!5EhBiTce?=jwB^P{N#!(Zzhd5H9eN1Ai*KjEz$=7}2^ck#+Y zDl0>~i{%`d4W^qDgO>JUvV(wum$9pwSn7+*J0;l=JZY58cOvRsjuu-i!sAUctnQZN?f#4sr|!Mlpq?fB9tO8ierA{5m0{>=#zuj9BS!-n{9P zCo>N{5jpTK5*DajdVFrbtL5r>_`{oy&4yjYD7dG6DeF_ZHvB6 zE$ka$I}yF1MJ<0>v!;%eFL(-DVTo!KS@DQ1QaqwmNCB(Y2x=5|OUS`tj$^Cv5Pg2-YR1Q}(z+ zfE@nl$b{{87^&y~VcNeuJ{98K1~KY!#I~yCpjCZ{=M{Kl1Q*W{6p$WkuGdb{Cz7Jl zWnhR-PNsN7jO^Rs$65+zNQT*W^5i(#FR_XMYPGh=3kJepj1fHX6_Z}s&9+Q{@2>JC^Sf}&DmXSP{n-R2vD+m$xPNrCQ%f5JRtz0V+ zhsbx3X@WB$Xg8VXWMOXN8KRb2>i8&gZ?ZJ~nH@|?=#gnp!Bfcdc$p5q|G<~{>zcP0 z?Ik8(KO!D;-OBX%1-xT(_;t7fh^^-r#I^+nkh8S?exj((R1#7RlMfq*S>y671OY~M zgomTNnFjE+@pFC1b;0LK`Ki-iIxTb_#*krdCWd1ennyYB6z|N{ zZ{h8>)ahImpzJuCeWZl2{ZXgzN_NZWY6WPOPmU83Nn|f6C6y*Ad#~8Wg)K7a`AAxd z=f63R+wPo6OcT9WMQOU`DjB17uz}9!P@)fqlauQT%-slx@(%YtH}9{Bw(F-r)iU!k zuyEqo$WW1W*9}t-Q7^NOvH7h133D%KI2WV1Ml~15G44Q*mV*Sze5YK*P+fANDVngH zgrxQy-)OO+!Fv@q$(#3+NDVfU&pkbEDU;&U`Gee@HdQ4j9lnc}e`Rex0y!+ODs>52 z40DSifqy8eFA>;yV`xv9p(PE$i>LF|%%4XX`~g|!1gB{6Flyv%S-uX($Iyfnhr2=z zi{vFovq>W1#sHcg_*cfGU@BEFtOg`7Uj2f0OzYs+?=nG$-!2^V$`#8r2GH_ioLgY^%PoRYAffHMgX~Tf}x)<*?Of z;26HD$DBO*q((vQ>k)|hZ^3z`@FYxPaVC!6?7aKez1m*bb%6B8Rbt*~FRrf!KQz&X zb9^%NQ@#IYx}}`#aYZUZ_u;A({~Ry2d1LL7Dp8~m`loggE_n(=St6Kg z0+7R2Fa%dg`Ho+Z)m!SMhqVmX2;!I5J`RW&LDQCXO-U2QN}&)|8<-pr#qKE=$n=_E zR*CM?l_8Ult6FxF8?cr|Y5K^kE-G6vzR#Mg$P`Q+?4KIdvRA26wjx4lO%Kp~%3IRC zxC)S_R#?_P7YZ05QpG8)>+}{=bE4DvXj91io&c~ilARGQ4-xs5{{Jk1qva7(<#_6> zuHsiJ413Js_giS^K)O@8SP9LUX5_|SZTn0#4HrCYCX_^8VZzMl=Y!&9ezW)o*M3x-Fp}M-{i*fSSB3@zd)?u;I~)D z>6A-mDuX**v>g2q+2j?TBj5fk9}8vDS_Jm!3)KD9HQ)?MmoVX20XopXob#>-WR;me zIvdNObWKG&ml`UjkB{pfZg$UkFT+UySkI;lF;wM(pM-Er&bO`sdFt)BmCrEyp1Z^< zOoWhN@yOW=QuuMuRhKiPXqb#A7!&LOP)uwNnP7m@7cvxPs@hY^*QSe}?s}(^Jo7{) z8730u&w>>o9k}lx(lyScmdAm$A!3d*Ee8#|^IG{0(TIz)dju=BW0T{Q7ZV^51&s9O z@F(qhW-5nP1T!|=o%FkjUWNlPJt=L-W^3>cWlrhiDd6{Cpc2n=YxPSRC7 zhBst!wiI6}N`tTWg>Pv$fETz>wX}&sO~xrz1_J+AI;3#zI9$G+*L2?PpFgp~T|h5v z(Y5_S>_M0e&Pw)SC$IUGsd4&wXX*G$bK^Pt=_z-=Tgg(dzOs*H|5g2W%v!NcIHX-w z5J^~bQS(ZxTgf<=>wM>Sr@L9TB@+b?dh-^I zOkwuL`ZZCu``gv`_9UP-j`VRM3eiVC+n5{FRR{*8Nsq9=7?LlpKEi7aVVh0 zem42QH%3iHGmxK`JpV374ZPPVP=&V4OcxrHs_g3keUj}ToBA)lIM{K7FePYw*=G=u zIVzJhWKZY;_|m=sMB>94`&6ibv$n9JN|nQ>$!c<*QAFR>(YlQJy9uV@fU+>qlboHrKzG)09(=Y#FmWpHv))8(xG^sPtb=@tkeO5#Dq=kfa zoCB5iK8#|rFD3h~ew9Y2Ftx_?I%~y$u$t@7Zc4JHU-Q=$pERfb#IrS7_kwFpVs`7n zEOvy0a^+UgD@8knQC02vY^jsD84E@#Qc5xN%jqD?Efk%Qqis= z38~VpF|VX@_RIRY_#&!zop1RGnPRJS z{JW-4qV`6;W#}A`Q7m(UxJ7h(rpGkb3w{#loj4J#u$EWc z@SXWaXgg&%F?WsBKS`3l3cA{IBi%A_J82L^FJ85fVo(b4m#x#nVB zO0DhGkZHbwNmKpv6El%r8SL|gJ9`9bP)PSqCO5o2&-*CC z2`*IHskG?y1-gn@ksZ2V{yL8tTo1tHv{_B<|{Vk8P*EPGFfxcJQ;S56!`ET@U ze_!k9c3FH8O;6jB=pK-*6yT~by|5O zIhn%#>!&NWlSNTtZlKS#_^+V$oSAv#p2T*45LV5!`zc~2iJin#IVNhkATW7Kik$n$ z{8=^`WkbD}8x5rnuBF@fqURb>E#c$`P+(_Mp{6IJ)1RbKw;fjQQ&-Aif!?cW=#YI@ zqBSwp(tBr&iYM>y!nWwU91Jh%kepDUci#?y?((ZWy}ivpQzJHO#dAcG zl>X_7>{h;yNWomRh{e=#3X*5LuF_cYW^AKl)KP&}{ zp?#p;G>=~QE?sZlMReA6K;#9t<2+PcU<31=?|8^jma=1Q zPJrG6B|Xl!sL+1NnYKG~v!qQI0jJ<73-oN{mvrrMi^lXwmnrs-IoW%=qxq^AWYABH z7S*5yUWBUXvLn;lUoX8;s+3_hi#G-7#%LL42F^-7A}Gi}cG0P49R@HLX*nz?A-W#i zio+?74`wv^N$o+{nk?US8`}Er(C3c-$RebO&!O4tg>euko5C@HjkLpQ)7EETh4(Nq z=s>8_D+5BI%m&VW0vtJUJK_;-%jHZEDlIGd_`=RkcqjKRsW9Mq37dBvl={tna|Ggz{y@nu zQfH$&Fqqfhe>XiV3433prjSc<<%Cd4=!wFoR!6zo{8%7MXMXioaoxVE%zsLGV*t7D zSj-;*Cu(EGJf7fGdVvs%N?a}G?TM2Y_(1-L~>L^@;QCx>x z#N&CMY)B11*2EGMz;xPn2n}V%;5_=q4;j4_S**i}H75&ph0O2XiNk6GO__qr?bHvB zC~+8qBskX50X-HU^UHE{grieTUU$MBiG!2))r7okw5Rdg|8T%666{mrRS-B&D~)1J z4Yi1!KXc&raYYf}_=7?^gxsSsl_c=?G=3a|f#&5o*%_9xLVuEij>zTAU8Dota(cUQ z+*u+Ow5L1alm{sSeld=#NAOkR;5dkE%6%Y0DvJG2qbXgy887+N6DGVew(_s|8T$|0 zwgCTg7i`%tWQUp?t^C?sfoS>kkoX?BsB7rf^Ve4RQrZyaqO@}H7yLo-zK8{M5nmr# zl{n?8>6R{DX&q&NJGs55vRfa!HNc?xr4-kcH>}p>2;@o++4V<5)W6b;tI%m`ZcKKf z1cq-n|11rk(B0~Ho_H6N!)ib6`!t$!8!y>~EMI*!5U*lH_UpBYW z5%;&P^8p@VtfCvmZc3c2cJc-n`mW+7g*RVKnmZhTOA-TFU_n9-ag%jZvcr(gBfs#2 z+u_j=P!G=Z>B35p;SSWI`H`A?a#l38&*dVw+Tpg0*?PQ1;$rAxK@+Vs5ZHD<%8Yr7 zKHv@NKK-u7NA&GCv`ZA!>T|Xm)o_AlK9I(|J=K1YtZA&~`B19~wIDE_edV+ESHC0i znP*{K9=n0ZQN6ZcudqwPE;{}i*l0@^j?xiiSyPa76M9EEP{~G;SnN}9%c>M}HW9r< zdG!R@!e6qwO-M{Qd#pa;`((*VTmP`CObkyp09i0b<#{anffyPm=?Hvsq5m)TGJut~ z&OC6XK|Q~Xd&a)NjlPpFJ6aen=4pQavO`3%^-?|=Pg|xiByfTiuuc$&SCr3VoF$?P6+{)Vn#aOSszZFxj5NZ^oxMDwZy;_OnoCg5D|-A1RR#&n_FbU;X>+ zQ+rT9`C`g_?Xc-_*THeKq${kxNMQ!8gztNujlPe+vTxH;nt+G3YcqGhUy4Wljm>Vx z&QHal;1_ad06akf2J$kOjPlYg2Rm{40+0E5O~v<;&{P0$EVeYIv^x|e*l&k&A?AD4 zsGcB3C4dMO4xy^c;$2Ihyu!0zgvhT?46wesgbOK(+E2az(epM)o*q%6^#`5T|5{rt$#wmzd?)RSKU9ScL&zJ>?m~*S4DXdx4%#?_&A#)VD9R?J*ra)*6Pe@ zlRMp^4-f`H3Bb+j$!tGw}W-M#)2 zQuIEF`+WbsblIos)>NlkkqUCy74Z;qw;=<$@z%`|k5*{vSIx&NT`BX29`==VoS#JF zK`Je^@06$o=3U3t88r19vmpJm&S}|_q`VQABt&Q8LO+{c(BLs46_R z=a(j18xq@Ao0Y=aQOM)b4$SK=k6r$)rn*+2ErZ@nSt{55j*W{pkqJ>#D133Rn0g=9 z7FC9e&ml#ikTzyxs)QuzPvRB&9U&z~ly4gMqZcY$Ss9S7_3%bo%mpOveRiba%m4fB znlE~v)lZU5XaP;VlJb-~Nm7aVXWDc7FDY3jPjo>-+&AObLiI9-Xh_nGZ}(c? zh?=?3k-i-4wU@%{(@QWz&rthz$Se&vT>; zEw%g-$F4|Wn@R!uV%&zf+w)qRYm6y1Kvn*h9zl>`LRX4?2S+qJ*tf8=XS?Hg1H&o& zzmUj)g2GG102Rv>Dm?xHX5A}WR7(4)U223*0G2;3%;@TbZTfxibY*8#7arf8+0-HD z)f6`0YiR*=zy%}SAr2`){lo+Drd5~I#mX8pEMoZiMUBZ@e6vXU5Ozez zt5!zN5I3z`WlTktaQ4p><)C4#TmAoLuRA1%^l%b__=(#td`q+cVe1|I@_hTp|J;>r z*Rr>;jIG*g*|uA?W!u`8ZQHhOyOwcf<9GJ{;J)wA_xA_%=)BGY=W)F7e0B=|DZ<*@ zfh}B84M-#j;5qBjB{tBHbIm_Pt?1HhjV}3U7gm)KcDBrpV2DJegO2V=L{sVT+vR{~ z4_)W5-hw7e_D7FcL}JTWS$jKZXwZxJje`DkXi#yf1+F zD^P&Ag4>8IX6~sc9PbC+J6XIdPm#3G+fa`zOON(RKmPL6l%g#cw8qJCx*-)rRGx=( zY}Vt(|C(P}2JNtM_hh<{DE?TA<$oiB0?G=f3gDh6|JG zZbe;#aogBJlhWPcYCoRv7iqMM=&ijA0m>+3NJu@?Uhr)>VoNj*=$p2JZ1!&<`+>Xl z%O~Z-yW8$dqqwjh(2Ui*55s(k7=UGHH0C~giDYbh{&RR*#i#GZ9*Hl!fQRxZTnQP! zmD-_MHJP}?Nci~9XT6UTv_4AS)g0&%s|Jx9xqvjk5{0m?!evBRH=_F34?C-$r@wAi z6TsV<>@xv(OaUs^aJ5>A%OTa$|t{parHU``Iu zb4|bBdfF-9MTTee8IgflfNOcjtKCQWahv!E*I643FxlK^VwVA?vaw5_{PN%Jm~m~d zd0-VF#d9}xkyzn5NSX9Fz@~9FLe&ONTD9KMs4ZMXcs8$o0XUo4Lbs1qmoc}N$E|tm z>>e|QUHriuTpAN!U3d&u!|$U|zJ}o3_mCP}jH}{OHoF;S>9@S94R5;jbn#Fhb){g5 z-L1Y{P0L!0brV~iCI|m?eqA(__fYKKg@1~xE$MwhfvX*rq{Ex8`@Rfk{PkoxBb`73 z8Oe&44#PQr@KXlL!0rO14WqwEbjz9-~?A0szax!P zNbAtD2zjH5=!DvEG4{ojJiMh3+t)YD7vzOc{K9RR|CT;9R>J?-cebU{ZBa1F=2;o( zVa+(yO6T5PZn(=77~o+qw-BTndkXpaw&MYePuZl*CH!w3R?=VOa1*gy+u(x9NCeLo z8b}cZ-LJmE`Y*BwDp{R;k)urNQSt9N-2^~kA5D=+qPC#P*xCN*k7zgID}AP|G=udE z72^Ym;xS!)+rbLzpg?6Lk`G%cDgX(q^|`?`bwfkzF;=f6|5UI5Sy@~v6+$wiAV4L9 zps1}Ns?P#w%}>vo%~pL~erK0WRbbTeA7NTJKA9`=_)RcXWwl$gJYKZFfq-0Nl$uf$ zNChvTlnYxYg~is9ctxEdZU{u3&MX7IuzC8S2i@YotNY1X%_4fP@ih2R_AvsL=g-nQ zXIFF}5}jiPj5j%zYFCo)C$-5pZic)T&E{7%w*q)kQ+`W2HbkHWl8PjUA#&)ugJF*CuTB1oh*ox>Ix;$Z*l5uv-_Z=dQ72r zaY+|ptGbYpNUPeMwg`DnNv#{25hQ;85{Kuz$+Gr?&fx>WNbGJxX!~7)C)lZ>e2|6g z`SYx%KhgIGM5DwC$2H=l+lA*E+pFzL9A9P`zW5?e;DT0t?29cG46zG^f4nuYNk7Y% zdgMO1ELmVfbcc>=R~Y^o(~*&NarkgK13sDt4U)BR!M*HQCEaS@#$MmgD2n=&T1lAxcP5;(P5x>uI4%z@EYU;x<{nty$Z%x`P}L zKaU^1WCf~a)5lrOSIq^$IVX~yKz%F)gdZCIKo=cC6IXN#joWUjf?$v@no7;t6V#7m z-7V*#Ap=-kWi@953S?tQu!k@)v=;NssCCy6NQGtR_eY(o{AFUsHcVQ&}CIUDZ~*%&H8mU2}Na(h;bFl(f`Z zt*bX#rKIkGFCQ-z$6jxsv@(vTpDXQ^aUstPuHad zPANfjBtr~!SmF3K85Ypug}H0XCP}FA?eVg#rd##Fqlq}TU=xbm#g{_JMZuEBC)N1M z>C*C;_*wpW*!~-U+1x^yAc{|BLQHIWd_VNzF-_B5d4BmjxP=jRek5Ut)c1BOD?W|F zd0t{NozyqKa{gIZ9wZK_EP2a@m-Ec$53iemr%R8*Sdm!u!j&`liChEY<&Rv_IQ3}d zI|9E%y!T~VE-3hy`@u%REipE^nvch&sHrd`vkH{tcLqu(uw$h~$?0mq=O zLzb|snlvZ%EA1t07(`efUwLFvf(|z(XIr125QNg`qRky!7(<=pU6cf6+X)@RM4N^o z6|gj70nY4~vH6!k7N$`?Ce<-4>yd5w(&Ii47UIMzg)B3b|R%bBO&W^um zu&*%|L(q&qAP*366u;cX?h11Fz0RxbZE;sY(O9K)!9ZiHm9eSd!$lzxjppulYW z04=8HX@!5cC;2%N?mbABxR)s9YDb>K3~oM<{$>jc{oV?N=G|KvT$FHUNuTV;O_*G^ zt{3dxH3LG?VlgT_%C|zt!Dqkp(6`VHN|N0g zfC^J-(q$?3WZs-_uAq&2o1m!DV9ARsbjH>su2`3%EA-en06(80x%x13_{2Dq=4+>R z*SL`=;ep5(F*TIlmc9N!Dv9DsEL_Uk7H8qGBEBwZU-EkEVP5url==w zIk_^!m4*>sTH}7Zo&%tc*Yrl*CJg@6=|t~HyNuU%RF}yk7IKw|)cX7Vm zU(8m$&~Ccb6Ef58WCjygxj7J?Ts{$hilH+OfJhfwWsZzsy};-{o6S~^ z2#S>uJGW(@6)6TTYkvJh5I6RL5Op%Jkv`rFvANpl^waJ~b3QWM17WS3!MwK(JR>B8m8&HNNVTqz3N)@R7L$U{s!OK8&^)vb|B+ex9Ah zn0XE@)-0Ou$;UQbQ(nTYG69|i+`AiU(`VQ9lfYKg+#4q$$yjGE@?hVFNekD1)BBNr>!}(Y>r)|GLG+}0ci5HrcJ9@^J$ai6a%#v?57N)S;-`i zR}$jWhc)3(u_nj4_4l<-yZja6Ku!pwv_%=nTT^NM*9@$ zAD;pPNb8L@Ixg@+hYoOobAMAwU^<&-K2iOmDUW$wlSF#EVrhUr)59Ur#1x>212^x3 zztOUF(K}60z;MYvD|>GapabLg-+BvjF&Ol;((T~A!n?32^SP7xiJ^2aP)V!N zHAB+lhBk1V!J4^987^3?e8xOcpSOMWGaII$G6$MO(EIIgYviwPiaAb(d}?}`Z}Cc0 zGxEFXb(KtP3VHeLb9Z#TNQ+HOM=dmNc4!aLU@M&TPEH&Z*8CUpD`e(Z3r=F|r>>4z z&S&c{^3Sg>H{BGo{h~5C)6zITu38&ovm=NqFN82 z$c(92g`A@S0tn=vtudDD0le=+R(yIktZY<<1JW)ml)UL)yL|Gn4RQ+}H%1>`Sbv-F zrTCgeF|G878uDg}<`rL95{Z|FN+25n(Jy4+#Q`1N-W>q@PYif#*#P{xa6lJC9|~q@ zc?rlLhFtI$oBkV%$pyH-fqgiIQm%s3ZwFiUpzfc2>YqOc0PDh%ppImNK}*)Dl1%Q2 zCG`)KmZbp0y7pBs8`@Q3x7voqi4v(oB2VM2X`|w55qunDSAi!pv5m8f!vv>@D-2Fw{wIR8OvY zij`vb3a!k5^s~m@iXn&B%%{vyiR!gO=4H9CX*h-H?1wXcwc!@(z5n@fZ?EX>zlg&O z<@W3K$$y3WpE>l;RPsTLfG7zkx)ls1yg8IcN+T4p3tSf?QmY{RY?&vWH@J75%ga~o zw1mU0n99NjZ+w}{$$MsN(eJh_Lt-BZUh_2t=w&BOGrLorZEiRMnLtIIB9Xzh%J&%^ zUl5*#ACBh->n47-5<0>axf=gCj~981?^0s_JY0^CaI3&gW@(^6C#l;tmTG+uQeVNfj>VU`hyuk&(h;q7u`3I~iOe>??Ty^;X7;}Ft2ru4+Mzq}~$sb-()^s1JX8!eamr zpSU5v_Rv*aAinHa`RIqf(3>8f?Gm-NciNm(XbT+f6(~Lt*@F}5i&y^Bzw^)dVEYwz z(8d1y=l~2#TqsVB`dntPb1;tQ-oyxB848MkI`A1&AZKykL_mdiO~t$rzy1M$ITZ&( zaX9uw+JiPmG|O^;s)Wxttg5$MOuFK|uO8{4g3e)~zz#f@|CgdbSkK2b*=rrW@Jm2d zg;;y_TUeuTKULLyGGQ+keO0|VPivDd6$b{!Cp18K6jf6(TPLW<r6MnErXK7ORlp&w|nY|Q|NN|-+v#kx0we_x&Pmv#06Dbzv{y)aP!`+F(Ny$Ui(w= z>m6qWlBJ6mFfURh9hxID>in$pmX=TO6VO6R$J%|#H^dgWlk>FgC`jbW30t*W=Gv55 z%H}IgNn7Z``dVd%+aF>8x|#ymDc1EG&`SJP0`w9Fv)YZW4i-=2*)zI>OSLtwQ=c_H zcpzq$6UE_9!l|6{TB;Ve@e*;NS2|^H$DqH4DYqN_Vrp~51&F9f379gvQ8Xy0v3Td7qs87>Z4=}lb~2+p*C z7h$OIDE@s3zD*J)bO3Utp26}R@!u99;4KOSTCQG2u5VZ3artlYM@b!#zSC)7s@-0c z3+3xZE=45)rVB(g)l&1z4)^cn4eBSbwP!-vEw}0f4yz`$=Cgqp*)ba2US$e!EG}-y z`cL;t1^Vk9sA&A3IG3HSws0>ICA*fE?d84L!@h9+^H4)DK3n60IrrJ5S-E6wSe81a zb0x5xM|Qjb8Xa3y&FcSsbih%hr=G?7{hu%OzbTi5sc)dnJq^5eT)6gLmd!dpME$Au zY;CtzIc0V|r+;gcp1BwaTd>?_+Tk%UU3q$-sL`L5)VO#EOVd%u@6fMj~jz}w*I!0UM8{-B?JVt`P3Vy$db4a>P;=>=GHyvzW?@?#yiX)R5l;CZ6I zfWd7YkYoXyf*b}0-KI@f7JIfj#a+dPb>xS>Buxhu{c<8QS7a~CqJDZxmw%lK{%c2X zp&}3MZ2825^8YNmvmP_IUgjZ2LqIqp&mKg8l7)$i$9q(EWZr4E=Jo5g+j8bIJ-~V~T|2yb0h)}@1$^tm4uXe8@D(GctPnvqomZmbQC&St{NAd89W4)x2($w#qkQT2Lea^=sg{C9V<1J5`H!E z8q{S&_7b64wcisJ608;#x~!zHYf6pc(zNTX(ILXqwP`YSOc7i;F1D`;+kLl`*R z2sFO9q;z}USp!*-|7W7hVHEsj3veg{XpJ0hH&#iUmbkTm3y3K>=RapY2nsd~0;UDT z%=_4;ncXU|fR(?9kpE_9$^nYunzw!CH^BKMV#DpvUDDIE$_tA5mj&|$fF#Xcp^$if zBuV5l-Pv0}EluH(qVGHQBLo+31~=y+AQT3CH{rc1?ysEBuE&sV?!XA<-E^nrxHg{x zTm)|YMZEM0K*5t?toy=kjqy)5ksVO$6b)|n%L8hVOp8(OFm=f)fDj)Iq#7h$;0>bY zJ}R|3XrdYmcqb7%{Gl`4hM@UBOF>w5~u;rQ`PmDozA0!6R;h4Lm~fb zbqI@z>3K|ICJ7wl;$TK{S%FAQX8WWIM|0=p4RMQ(D%=-L!S!uHlS2sWk3-6UhXg3I zhMQfgsWaXEalGc3nG<{KzqWQJ!~m;Vcg~sbq3fTuoBeT zC%dJCk`t(Cf}c2@$-1n}R<{B;o|BLYfot?4cKmx_z5jxMbjGOTwfVIqt2}Os5Nd;| zj**71pTFtSdX?>D=*HYV-E#lcv||2R#4FHc_1(lDk^kIvWAFfpftpTQ%+>!9U4cO* z4?wr8%Wy0~clYd~3-{8~5?{g1+;XvCscuz*&T$j9L&4^{RJrRg*UWt)H>lrz;?O+W ze5^v#{QQXZ;>I--QfMVFj5}sxe)e!$UA9#i1fPT?eQXC}etoc07W4CPn~Zbf|E7F6 zH(Vs?tS!S^PFAY>QKmY3aRoqi7r2n2;w4Q?P^v)xpOq_&{sqB@t~cYDJppq-A>u#B@Lo7WC8)G;JDc_5 zMc!t5WVtMhQ;`QdcTw=NE*~Z3?ycPi|#P9=g1?8`L!p@yz>E#Ie5k@YrE;$aVW(0H=QcutdPn z`}QxA%(*ie+$Id#2;uRLTJ2ZGB87ZXigyON7PAEku|-e)4R!;nS&|}R)Y9k$8K4Sw z2qKt{>Ig|Lbbc49xiV^ho;A6am^tPY3Cld$5M)NJr}bO&tq^#{{(hZ23+W_fxs>;4 zNSV3up8S!xt#{aN@NJ(OB0q61P_@v5^Ovp$1eIbxefh6b>Ay|s9Rf&suRrp- zI#aK+j4rq!nA|tCH8UNBo|3#k>7bQtft$lr!CVDW&%Y-Ic zHJxZLNmda!52kTPXUtNh+&DTU}JJ;I`fuP7-V#oD( zWD4Yu`17xVo@6*B+9b7fBvQyMCk%CFVt|J%`+H>sVs@OlPRZQnm!g3#CAGnKl0>x6 zUtYE5!%Dv2r-@}eHSOt$az>pmKg{&E& zEcmsHkkg77X|=eC)%1#osn=(?s1hvKS%l;BkQAFN#Tj`Z=`h}yP(-pc=*Ud6Z!-AL zCGCGN7r?oO9SYNsErLv@D9Q0nYBQ4fLHYbs5Nd9ABrk)q^4G?HKg-7;!qhQqehVXQ zOrCS+r?GI$x2DgKi2I&b_`!7Y7)kcVK1BVbVr=Fjpq{?j{l38XUVEkPN;k;~u-9$& z+dNklTxr!k6ndR2TlG$b{VblZxD;h;0663F&5kE#hcg2S!H#_fyxz@i1o@6RP?600 zc`HClZ}J39evN)L;NTQaVhq*?2172cPgdwNX#vC_a^Yd+og zTSDev2M;|wsQdeS4zCw?i-igb<-$KH7k|qXcHY*#FFE>XEZx%?D3%>Yr9@2x36Dj| z%f)50v5CQuP15OKH#%}ZbnjS=Fp+YF;`w#c;${I4w>p4}K)6MeCjY^FsCr9;c0EyHLU>6vxln-F{RoXDo{ySfG_nmDg+cIOJdN}qa$ zYqVXV8oQ~%mED_hl#+5YT)Bp2cf7F@k5{@H4yN;qukxe`r|13Pr&f~Prf0e}r+pu- zQE9M^D98*7>fF!2FYR5@&t!~C*U8y-2clecK%>b4udq-dS5gvv0TAZ#zUivIFV{(a zbUw(9pSnBW4ugg&%tj`!xLtEww(J2OM;XqQNgEqxOJW1)cB|a`TC7r9K%QZ_?`-*g zebF@4L8YbMZ5KRK7`=J}gb0^D$kM!P11jwg1rYT0j^;`WfjsEpZHnDkrtxr`u_#_H zny>H*C};W!Pc`d@P}`K(r<0}H6PcNE@Qim2>wwR{Z>YxTK8MFF*5#G!Vh*fH(XqX8 zACGlnN|BDzoCugbPSh#m!$vr<*7iOsxy<}s>xa$+&1Xu%2}w$&d5my-arG}RpIjUj zc$N#sXjlHRX|r#tIBgy`U!6}E=O`TVgfCJv9cz74S z0EEa6xp7`CfOe%D;`d1=x4hcDR2)ZY92=xNO?YL&QdrMVm*a@dK-J&Jr02aoto<5h z#zLv<3#IR8e%Py!mt~rnW{|@jyr$Y;c8EBZ9Q5}lgE6QIY}VUq09RnJC9nQumKcZ8 z0Q$AlN$va;5L>2?U;PCHgF`62uOXq#C#G_xu$aY9v0&a4`T(8i%U}ijnu-cGYD@53 zspg3*D$n0va%`&&KeDw=cp|e?L+p2ARi^;P^#1-5s-v4AaK5T71GGUSKcr<~d;swv z@2J;eXZ$p;lv#+br9N#;abRp)T8_miFK~ z!}%N>8??-omZciE_2tpO!G>W31eew)*;;F2S}Tcik5_gD`t6T}S~?`{Oy-GgcLx%b zOR<{yq$lT3!8zM(;@BxhU4Vq~U{+OEIhW)6QjB%uPTV`GnB`~@^0YnqC2X>=>Q?kF z?q~2c$8Fy*ag^cX1$B+VU$&z4OGydWru6+^_!amguTB)D_h$kcoTu6rT&6VFMcV9g zm(@eD`ANil6*q}{^4Hx_c}#V&Ix7k_ zhBt-U6}VlD-89{0mjI^@dd{!+yp&{||NJmJCzlpVy~O+7I}ik9;`YmSM#JvcRNm&B zwcW?%cH_$Ogq?)`N-z@%53qdD08drClHsP~Bakr22T<+djc)&OR-wdaLYR?ikWqAFSL9Tb#5X7e%5MHA zLh~L1@~s{)JC$g^nH-+Dv%wbBZYa3*DsHfuFN9FYQq>sJ_4 z*E!awcPq=df9qBgUw-^uHFcf_xEsSaP+(!ink7gVbLpJVd-3uOqd#OxS9GO(FpWGM z2IOb)^1hwlG>}GlY)IpK$gw;1zGkpE{GXF5AE(paae_5oj9@#6*)7jps@W3tX={~IjmZEw zM{%An1W@PvQB!$$YM3YAox4{td^?|D?aa4x$O@NyNEB{}yXhKiFKIMxd;4q7QZuhS zyAflMEQ>gYp>a|*i@7wt(tGE8_7r#KIyv{e9u8+w9Vo&f;z80&Q<2+gQDNL8Nwtv*xPI0F z@$_PUOzIz0GCYMI*)|&bmhUEwHvsT%?RhzwvzPx29*NOhkEUGi{`iV4W0OumJ^AO? zk-v>O%=>_wv^fP(W2kU%CW$R$mAgv?TD))oVd4aL569hkKUpjhR_UafQOF}Yi##rq zn($dRkg)sLy~dkdIA=A?F}*%bP`jPk`pfRBBTnh#DN?_Or67^)u&&qG?>yNJPy49+ zAO&+A#4NHS8s5h+4x!<%T8!rJg+2mn#9CFobln|p_lJ5f)uN}W)!LsTCw`i&glp1E z4oXFYci7fiF)&qx&#CmrEH4}G zG$}a|p*k_xwl9VnP$D&}c%(~4w~dCSgC0j;68aA&>ZVo#6~42BsuyAzRTX+eaSozo z1?9&#flRVurO57RvIHzFEJNwfouMnh^vG~a3L+KM>wt(|t2C4lMfNrWY;!cCg6K~` zC;54Qr1#BP1TtA$2H>IbN4Fvtc?a}V8qY?)r4}F>8ru3v-6rHENW{)AAJbo-;J+eZ ztwgT?5(r>h%a9a0f=b4XRz06_Wk9aU2sYkK2Pwf z?b|nBSyfxCL;fGB^M41neX7DcLZQM=)v7|uJi(td?m9}_PQ_O7)mvDf^KE1;u9xtO znqEa<@@{=!xErbqFKKSkvYG^3U3D0lLtu~QC+oy6Q%8-RiaDbM1`|zE^udv>mq5K$ z0np2;RIVeiZE=WJtXyoO>U*ajf{6`%%`JI-IBR&?zgcw#93d1!T?W=OM4$mjuvkc+ zj!zXu3=;;P2_W%QY!BJ^{cXL=mDG$In@qJtEwc~1jij5#J!+F6E6rpyN&flaI=akY z@DH`*fd2xOey@Dd?2gh;Bq;7zcIJb+KxN_1CC-|gu&5a0?FX_pN=s9#_KY3bRR!VN z#rMre&{5PUWp{vwsHXV)T1%q`@d-pYLqnZrvm-MQsE|}g7auM!r>?#U;E}a=>w3_x zHi-zOg+;O|`Q9s%u=o0O8C8TfPL3??PM4g`3b?x5Ldu3k&ckj={C}R0L0MOHbUkv1 zFX)=VmjrfXBW`=N0qES`e9^}&j5XbXuvVD0+z^eEykRxTJ)YBR;tnaT2HVVMnI2N8 zapd+8%&n=Ny}e?W)RN^^qZ?|^lYUYNLsb7;u`I)}a`)7F z0D)5)`Cg$~au>1esbG%5s_i_+34h5oC26<K4FJ8RpWDCaf6imNJj zI9cK}xpetpcpjACfo5|-atjD&=i2&wMZ$zY%(x?q=BJJuEt0VebHHx%myUKHq1~wT za$HEr;=V}+*Gz*cC!Fq~W?H5iyf*Nc4F{K4}w-jG49pCMB zhd%KrdOYZ8wQR!d`GR0M)z+-uURQ>*Z*$Q(S~_ZHqJadza+K&b80PmU(C%=i&_Z_} z;H~BiOV)TtOnZjyuYa?bkEQZ0-@~I4oBepg3KM$I97$ku0t~PUaKI&b)Q@QeK1XUJS}ONx zu7~u^PXM!AB{o3d{fCIzbwGn-Rcp04WguRojmkKuUJ`7!o{u&`dfb^PZ?eaEyT+tf zQ3e1X!w8Tc;^e#}A_$EQJ42(3tZqaNRslga?gs^ub)lz8t2bNZ98lYDxVcP+OTAQY zlooXF(T$8a``P5DCV&e_>;1+X2Aje)R&r-nnN<)_8IQc#{4187*~FRmtjo@*Govd! zR#~7pf#0QSH7(h?>FW+;-5a7b35n@*Ww~mR(jf0%iXD^6#uXD|T935Xw}_PZ{`#o= z;E79X&B_stmpSyK@v!j<&(&g?&yNPBrQq6;>M{$ZVT)gySK;jWKJ713S3HK{`#d4% zWL(ruH`E_w0d=#{_#<#r=|7&@juNuogD16~IwGG_Y2^wBtZ}>tInhid9)v6uuJMQ?>C)@|yvpRJYSc+~$zl;U2Be z={~R@ph-^pQs%%^PjDx+b1oiankh@h1}9SoB`MbVN6---Szz**55QTwSSp?c!<~bl zH3n&o0BO-a7NdRF?JpqvR++DS*?LAy!$*>^I#&C-_NMDB`5H|l7L$!N@@948jD0$( zU)MNRnLQ^y`*t>BGLn#KqujnRN>RR+uaE3~8+gw1^Y0f4o%U#Ze5t{aAMKur2fGQ= zV6w}m`AVL)%5{U@%wQ|$y5sj#2A!$TE2H*Tj?g#(Tzb}fz1)~>Lf(999v!$1$zUeE2pf?|MN3Yr zg$VV?x^IdeT15apn zV^w_aoMf_q!%afh7K*-&)m%ig2@x)eXBXcslU?Yh0zzcp31}(O3?gw%2!J15qAp6u zS9{3yl{P@1A~Px3LI0hq9lo&h=F-KLN;$D}_pho=?fn5)0#w7Ub{P~L!F6xcuX~2k zv8wpq`*dk&sesp7nLz@T1u_Y2A}D5d(1D{R$s>>p^)|ZA(#}4HRIt0qOJOLj)~vgB z$CCfU0`TCFo zamI!h1UK*gqW}5TMAtPhTi1*hAGGT^eS}ZeRI+-|fJZCm{Q(^GCymU9qIB3K41^@g z#)jyi@k>{cEg{%9%X%4Tl`F^B6glj?^kt9*eSjTlefH*qtEO}JO&7lN9hyR{doc!YmT{AZM#wCfUDyP`bFb1 z!Y0(3b_SqyUu>ICl})FLBFmjtlQfBZydM8%3O3$2pa5+ zpLsM)9S7q*vU4YnzifrE*1g#cn+2VCB#3|bdprU$7~?&^j;XLei}CvxBTig>7;ceB zHKtwsyA=ve`2FRLIu8hN(Jrb9vDD3`?86s)fqFyFC0VJmiSMbi3M%IEKKCnr_A_|r zUuoR}xRtrAHwN82z!?O`X)9f~VSU*AZgTt$pVY+_-{S|c=iBpTJqK3eiVj#IN^yFg z>b6`~?uCfy$KqMO1PPdCBfv@3GRjZ;r*OLr^$vNxKI%4|)GZUAs@O)5y%%z#yiCim zHoBuyZ|q(1Q=j-BT7VMtF1mjuaB7#S8zr0`kB85k$HY; zY2+@$JnyFkj%ZT3t$_WJ!CDjuTM@hb{rs4Z%K-3=OkJ6gk>(E2ZXJQGClNLX32nmc7LnlR7GE z)!mrnLR};kbu<(&_#hT5bW-mC(V=H-t-1653`FIQaLTFEmmn0Pyo;dgajVz-bWcgE z6SG^$6ZpJ+R#9Xi$i>>TG*M*X&bpxv6rph=0@w4w8|K74{W|DPj4N)`w)F|JCw(mn zWatM7(Qro-!D2AU<%?Q>NvBkayLtaF(>&x==j{jvC73q8^A+|rld+V-KJKk)iWz9l zTvyzPy_H5g)19Aa1ZoY!C6F?AyP_kvfQhpBJg72w0tiu;tth$T;!?5Ki%Sm;jZ!yo z%^B_38u%*drn3&G2Z&8Ib4=Lpz|Zp=)A^%^1^E7;x*4 z!u!PA0p9ARcxkIq!_>}iVS6P0=g96GggE|!1`!tPZLNf0=XoF9fZKOzIBV(qLP&bS z%v%oEhx0usJDZ8!0q;Oy`#Mr8+$KyS!h(ylC-nr${4`gFN<9s2#YCr<504yX6XcJW znvDf}Lr@eun1qJ###Ozc8d#gSsN*R9JZNb|OHiKpqOTjNh@zyp!9^Z{l58*kwM7wW zUkLa&8y@p*?BZ;rjpHVWQQhF{>Gruz=XgFpU}!ckf5nfoO3hwVOyFMV_l^p2BG_+x zJk16)&&dm#ejZWx$BP;>GW~AJ0^9|)FlAR~OxuaHgQrr*f%x!h@N9yMk{ai!oqoNf zsCM|5NSE;2s9>w(SX-1wy6|mhx<#h|fPN_u#xnmL<^Wsf)+yBA6vavQNQAB_SM#)-RI|iKKN(H%JYtv?jO5>l=I{^R3A<;j4lPnY$;x>xpDM&~)+K_%P|I z##hvH{4njN=xTu1Pt!*=VD8c4HYN`-hzaG%B|JK3U93g7#3E8%q$YFT@Hr$3$FYnl z);yj*m+tVDxkmA?MOb$S@CaEI!w-nP*oxZT z6+3G4KryKwKx5Y6J{PVUp5Cdlok|@Q+(WS03+mJ1hdEKzh9B9>o>VtY&DFx{EW)Ij zrfK})7nWmI7Qc}vI*&Ap{~tN&AqFT7s-^J@y~m{4xfQ;$kY%XdbWhblIC_roN~i6p ztBIlT`i$IiAv~YXC-{fO-U{-bHQla=NOPCW51kP-E#CVcX1AxS*VO&29}rIX?>yO8 z*SlLuJ~*Bn+2UduL$Qr)-7jlQc~hohVXbu6&zhDm&-FlW_tV&e)21j9w8;=J zU^&y*_ofn9#8Q*oY+iO3Ka=S}1mQ9TZwh>Iqe0YAO|T?83^z^d7)?KwssVeg;+gdw zJ=roI(%kX}w^Za@R{ip+h1OC#xTt8OV3vOXVx%Wk2|gB@k&VO9ufBHU8k*s(UHkxwU!|kyK~71;ir5DMWH|`E$|Y zRI_C+v|i)UR}^M1T?x<5e3{w7c}ak?m!(bUT>xjsd_DFW#@*Z5(pJ2HmpedP zr{+FAl!s{4phBq`lhena(VyH15pFYy9MnsN-}5e99N*s65Rsek9R1Wdq%rC5S?p=; zCaOYO2oYTSeBeA&zdg~$wTRwstl_`cRJ|vFWD`7_^!8Ba(0IMPox5Uzhc^iyc#1mY%(eb(NGn z2)jw8*(BCVvWaBxw({*+=?eI={+8^Os6(v!;r(ke%A3}5wAf|;<}7Q%vPP*` zrL4*T3xrBvc^K7v(e zRr!eFAMf3e)zrS9p1g9TSE3f~oE99EGkm4MW;?&>5czY&)OR6e669;7O}lhVQ&&_` zPDQpgByQLu3Q;<2$EYaWP+nen?bFQgNZiF-(mW8>kYJ5Bn#VSLq`lcEIKVf6Tnydz zr|`2O_sGYNM?7{D2QS<9ulkIG>Y)r*iByBfQs@g&RIMowe2LZkVEYtxkTzlR5UJ zQcbDnBR@Q3OLQAvIQw-u;)i$n#z~+fS_u|dI1SH5r6H;M>c5_vMzv!0=`wYFnBT^N z0@^r%iEP)n4DVSI&o*8{cDw%rzP{dy-BQPq4y|z*@lsn~u>JEbQ)D1Ke~1Jd{>{=~ zPaTnd#g4KhkpetlUb=EUenaalG;(*UL{SqGJT2|3J3J+}1dzD?7=2V;|F=+6 zX8&RRR<4Trpaf&8zVb1&00m$Ir`@AEjlU|5ESmXtTAezJqg!6bISV;Y(xTQ7*OClO zm#BY;g)bx6qF7l5f~5{fBjuZOzId?4hlH{`COQq-K7&pizCQOx{Rh8mI@|T$FH@YBh95|svB!@Ckeay*iyI7cz(Ub!e6`o8 zZ8rEJY9h}(I-wSpREn)=!U}Dv^!9qjWG21_jHg&QS&Ak%3ytdN53i3z)66zq%BqEU zXruIDbu8sCzc- zo4(45OgeZUa!Bs4Js$ANgwI*v8h=c?jI7h9Ig2*JaZX>c54{gzt||cH5y#cpN#$5k7mjbkQ^7Plvn zb^`Ff3$*un-yidL^7_<5ZnF|d?zF*N_OBp0*J7O(IPB9lw2ILcWMJ<$cUhULxy5BX zgOXTdEfv+a2alx~=1>|^X2B9IL~QjNbAJR#{^-2$-IG5LlWosbfhiK)w3u46j7ICcuc_X?QWc`RXYCUS-;5%b--1)$m7**QJ78-K03 zFbtFCbyw}q=diREXXZ`}Q?r*!LT{%0@+9zqMZe{H-!Se=p$~oXD3&{6HGTO7dw2tU zu!PTgMkV%!-ACM9Umwh3nMRg>OVbGdp!CL93q}cEa6WV@{b7~UKAFS7NuU0yy|y&f z{!#_+ID>GuNMS5)l;d`HZ6@axqR2zzxK(`$2k#T4O6N_mM%B;ZLXOdmet7(wvdS+= z*}tNg1NH$JNq7Tn-lrn5b7`6ihP2-9}t*26FS%s92kUR1-XH*@o;Z z^5_X`dWYr|Y8hPu?~QjpS3$Jw7WbFdc-$*!yl&UmYzCE^KclUm-D8$pRZcgIE8p-1 zTaY?H&2F?ymVCwtyJ~{$^t~S~o&Mp)0OQxBjSjboc7&waR^aR^^mOJI3fr@^#y~Vj z1j_CFCCbEGr!Y}pth+kLHZLdvOYIEhlVDOLW(l`7OM#1L@F<+&?SOjZ#1E+31Vru# z^h~mJdWQXcIn&U#FkIzUavUE3`dH z>JTtjWY0XQHoX_BuI!*7;!-GGv0AD$ry6apMbji{>z{jntC$=2ogYlk95NUAg)yWg z#frMX5bt{tk(~Xq8@P4*5EFa*3G}$GY26smap&!g;+26V%g(aGHuBI`+3Nx z7Y1Ma?(XS&UKeNj3)1`N)30K@HFxK~$^1L`Ctzrr$zqMxec8-pC97t4)~l{8txuj7%EQzN3#@bBeI^q({IRyDjt(yNY_sUb$v_gG85=>LC=on=s5YqRfh1|OW@ z5(o+I?m>bF9ejY`?(P;cNCbBR2{veOmjDUD-8Hzole^ga?DOt(>ej9Mi4PRT%v$~Q z)AH|M6B52?R9aUkf7ori&9Lh$1(BRDl(DSBT>(Fb&1=f8F# zB9H%)Zu8tckau@yG}oy+_v-4X3y($j3uTvvkE_%luG(YtU)t(061FWsmPJ?3<=&L7 z^~Cc5wyrVRPtnIU)CF3-##~uW65>fjP{%;hBdZUXS$zvXeyU|(Sx^OnyZk=ZI+uu2 zpH!vOgL6YOQ9_DHrM5P_2HY!AO<^Rh-id^k0XBfA~)S>Il ztY4MuGLJz9l(DBm7$>-$z(6H6N{78@_+33s_~yDr01cX>W%}pRR7-KnUKR-dq1t8T zn_52NUUkRe$vm=TWHjuGl8#$L7UOn4Pe4YvZ+C#a_YG=8sCBj8x+IFnoi%V*8P~O# zoYENv@kqBbQn%MQE(5s@5tIxa8>^kq5?y#p|Iui0lpG0JTfp!ATALf5Ea{D=Oxz87 zJ?-{lFVF^NkYM+Mx2L$*Um)$2Z|}sc26+nRHpujpEPN7dJkDn~8MC8EMBxi65wvEV zC^ZmJSQ=v=0(1cm1Fi0Ml*WTsVIqvyo57VQK%Z$8=)jcC$9`_1d~4uPZA!rMZ$X9R zD+cv?7i3~Mo9k3+o3%JyZDO{E$-`aS&(-@kOs{n!-q}CbG~oJyrBj+iwI|0L>TOcD zx=&ZV6XsQ@8lR=rwwb8aDwX%v{+Q|e$<0WGA^&-yUWl=NO$7fwFRkMIZd4TXJw=RI z2~&N3=<-wQ(Mf^8%l2=PaoxG6v}IUk^7+VYdQ~qQQpb|whjDv)#mOAOZ)n-6&FDJl zkx6VgnvbOW(s4>)Id8-zyqKfi_Y8&uyVLPfq?XYc`+@mx7M5s0n)R_r7WZqX44zni z79A^dbINw10QKFPgVpdI7WJ0Afz?ocXRNGFt<<60W|%g#MkX%DHp)XsCxn<>z|9K7 zJ+2ZJCgBfDW(tK8u(u!K;D#pVkzl8}JtMX=d9_9~B*9}Z<_l-48s=s5;ad9<_h1a<% z6?!WZY~kQ<*nmHh2-m<`)SGQ|1&FV;^vShx7q)n2Qj0+4_k!^#P)+>$P3Qy`(VKP= zxn@bg7)WOX@{eeVZ$EmAinLqHXsWZwxG6`TKbqC&Tf#{v8^iQ|E24`=MA1c`O)B(e zP#!`9<{M5%m7)|))61Wfnj0=~-@8Q*%O@KypVzw^@&2)=>@{nB9Vy{dT%S;X>lg|F z)rKAAs0%qeWd8B1>F+eWfy>eu9Nu8#Eqbz;$SlMEOpk@U1W#9b{8G(b9IW@3V-+7W zdKPy|+RylV=xOLVR=B@xkdJH|pG)g0#E3<>Y`B?mU8`($D{v+9yn6Y;!1}!2i)#Dk zP49+E_1*cVb@x$nq0W?})}PgqBGzK6ES!#oew_ zDLWU%Ct`3Pf%^+hX>F@S7By{L!MDX;Tgnr7hsueUi`mVFi9H5_^OmOKc31^PnguUb zg!($_*jTH0O)2ToNWxv6w~Jr(g2y(G?>>uzoR)70W9m1vgT#;aof%6|!z`7Hw8{cP zC`he(Bp$99j;J8=p^b!mrLhaU3hyyhRyx;{O{iCjc@&-$4TgDkgi`gLRD2%|?Es`N zGmd2+<{`zF+}}WcAudF9$!Zy- z^lLVu&jY#BhwLr|MGfQa@jeMzn$NpkL1MsD)us=jo4+XJejhlMEB2Mxe|o)pY-u&+ z8wLj2+V;!F^=_Ejrv`;x-cv6?+R#Tnq|EZu94OjM((!M-&0a0=c*noGj*Y|jo#wG0 zu2?mrQTLx4F98XK)st%-yz!hga}lZSF5(mw zXlEP5&p?j4}ZxU&JX;#_2dk0I)N~E%hup~ZEPjUu?vbu_q8|aYnP9#GV!V=wb z<5~mx`h^a8DnrlEC#boIW{d5ZMfuh%oe>wUl={nW+X&f=LDjCXFxtH^i$dLbX0LlD znSOVwl(GP~I*Ex*@CN-?#)stzRIZF*j(Jr&%f%6Sn#>)h@2pEKSy0sh(YTsM;fO}+MTA6G}B_4twi>anVs#v~Qs-~`uWMEZDkLaJ0)d@!u8&>U{Nm50B#IPfr{~o&h zm6N(7$A;L9I!~eqa{JkxVdo1(aTKDt6fk9A(_I4n{v1oumnuZgU){1nk01V?H)MM= zZl$0+@i)ih)tIV?O7{oZj6y&e76BGeYr>CMr(%tR!8jkF4X7)4Dg9DY{xP4fPhl5K zU`)$2VAzaIsfe0^lXcjR(#%Yn6ZFff6ok@|yJUq2Z3rKpy<~;=^c2@-6N){ME5#3+4#oL0j#@%}K;7Ii>i|(e zBbnXEo&zP&oIP>7C{5F}ZieW0cix@##yKH&nvYq{T)$ls>29m_lqQ$euMFt0I}hRp zf?28e_0uR%diY*7$GJ7$*ZQ)RwckL$Lvw=9X7p(>6g*p`&+sgbLU&B7w*H3h{6EpZ zq%D5??ZS{)fo@~At!>^1gG-6~cn*-tghbn17 zn>C(Xx4-qPA7)Q&Rw8tV52of7g|xk%PtTKxdC%c zWHJ)9z!?7dFeiLY(7Ui@0u1uLw|GRbEP&rG0((xTJk>fAi?fYjPEt9aTZmzqWV!53 zj3*FM0cr&B3=7Q!bb^`Oh)?x{hI$Auf3nW))&r@vC-)U90N9}FAm9W&YTP~f>k?gGOo}&E zA)qrcP$-TrKfl69fslK(iCPN7pH}wtyu*p&Gr^)H7jVpYUxns8el}}Y{+d@HN_Yv7 zqZgWqo2ho58@NCYrI$~D+qZ}bK@C)Pu3xkU-nk&|<>=vC`~*40Cp;}{I(VnYwpcpr zLEbt2hCDH;GC^^^vY_)O(&GSVYSVr18Aq}oFEI!E)*w1oNr6wYgC4Gp_4at{eK}M| z-Ecf28#a-$i6`4DVq#=BvVQt|bvJUry<}}>z9rNGS->uuVT2j(_Bn|$Lay?Oa1>_P z3D$*hg=kc#o~ZfrNeFDyw|GNK)B*HyYQD>qtxH@qX?6PMe3xoB>$(-Iso6S!Ia=J?!pL1Ny3*h)tHB)WsWla!)>cG$rLzOq9HzwL?T`}C54^@pU2T)u^#Q|!aP`*4GvD1E@zERvz z;(8n2Cx`XuNnXyRn4x|F{x9v&1m|%(KZ0odq1ZBWBz`-Pgm+>b^UAjGdfxXMQ!h%b zdKBgVQvFT3L;`e#>0}AZ02P?N4~Nt>^g#xur&)YUGITl|O21Ic8I;TG`;#%bke&__ zoJ=A%*}M}%@meLYFRjSFyln2Srgs0GPyQIoVNEsxwxw>7oHm}IcLD8@dU2JBU*AT3 zA)|`EBx-Kk_UFOzo;ZHbH3td9bg9Cq8dP^+-r_|FW_Zg8h+Uy(9p|LM$S$(wqG67} zE-OWG6pR2U8$KM+k6+_S+$}rfZKeK9e9MQhOEPIFA$@z$dh2-W3J%H2oDmELBTuVl ziR$UCc8EQ@5(o@Gr;+xDuu82UxmDUtl?BeoD5005VG_L+!?eTu#N`8Qw~NwI8xEzd zpTOjW3d<;9vgk;J!i+VqTn6~2t}NJUw!c`g+60>qxMXrHn)J(p&^lzo-v9GDls?Q6Y+J>7hxv8iA z;b{&XNQgZ>0Guh2!`c3A?l$bA4bYeD==fKd;dA|jkJ-agdqv{}+I-3>|1}{@7}6O_ z!JKU*=Xi53pnthUX8hlL0~@{3;KrC!6<~7t$T9mfXp6rF1}z zzNSl9v-Hzoej}-&Do$BTEcqpkZ{)N*anspJy%Yo8}dOSy$7{3^-2sXf$f8#@R% zRPR(jZQSI-`pExqg3RXon?22?Z$-=bdixC2ge2DJjO>sq7S?Qt?wg3bQbprnQW>+c zIGX;@eDr3PyWod!F9*lyHy^wmCMxrj1WnaSiTdRZgbHXCN`wNIA zjr5Q*4>((M&vAU3aqFl~arXcGGPFJKYkHhu)7oF|8k1&z%WsRAV{lb~8mWGQBXaSC zRhkz_ETwWU7W2l=Il`O38RiaSimkvND{VzQVZ6XD!&~=;~#s5YVNH`2;Mre7Jb}QA0$i?FE!it+ap6f z(IZK1l!V`vO!sG>y!>A`r@I)%BiFC>H2utfISXA}Gw$uT+o;ql5RXK{d61O4ahhBo z%sw!Go+VUv9DPapPF%mj>(Bk=+-?eq>Q!RF()}_^D=-3*H3PmSCJ{*6j#EC^w99JB zjWNa$}49uG!xcW!5&PWfa~)Qa&gs=1h>zwfo#{AirXA=!iTrbJjkT|5}xH%@%e( zid|qSPI!5B$|BM)w5}^(edawWy{!2oc55&KM;4SUdes!)8%V%S^4Q;_ zd0^zSQmX0Bxi;%?;dQ_{%{6mF+X%B{UCzS^a+FqA0Zku4C8w2hs5R%@wnx8QI0(0f zWG=fwYbh>Yj*;=hf|>ZLy^lT)QwUz2@!$KVU$1;3<9KQxsKPxLT`PIYP2O-S|BGen zpHa||s&E7psMv@}@4*ksDRo+peb?)FxSG%zmh1(ar61xcJg{yH_xe zq_bE;rES0XLi+7HE(#cJaE6QTvtrV&7G}8ULjIb@s`(|53F^2^=K$4A$?rJG<~#3Y zaOog5{lI$j_BW3T#62@;>9XaPG$QUK8LdCBUqe^()@NA4#qI53n_9)P@a^6=jDzy| z*;-(XPm3h#%DX2@AmuZZS;p@a$h~wN3M~B-nW>QWfu#Xgc+nZEcZ|je5U^%f8@^ubH3?ml{@xOeSS^?tFoO1QL}8k@0=| z0&vT4r_@Gvrp_CK7B1TpZRqnibA6m5zOpg*nh&P=DDIFgEzkN+s&T&cmN>Z1u=iV8 zy_z&|s)-O~`}KCWIQY3=CfqjXiO(FkKJ#lmvxbtTsq8=kB7KFDhMk?4|qc>DHz8W+WS5 z9}fo0Hu>PO%9wpfT>_3WBT#N^@lIZup&g_(Z0uyCg0mgt_Dsu>!%y)|GL+^J<0rPpn zjn?n16pA=065uo$E}SI8X;M?X;OnqJwG6=#NB((LUku2Ai(OJYMVGH`c#T!12w`tX zEmJp}rOEv}keh>|yXbYz0E-yh19^$_D;M7UP}<~uA;qDhKjBJd>?O~nIfPA`f0^mj zcyIgIA;Jj{zU_^B^%apZ^mU3`-@*ayCZI%_?ve^FZo}eITX~Mi)tlvp6M5D)d(?Wplw*qRfjG-#3Fe8ORiN)sc3q*_6 z`g>(#tFduA&u;oCP+$V`Bu->{4Zcw?K0RwL4wFb@dl2*-@8V+G<3GxGGdvYu8A!~` z7J!r3v5HjXIxak;ZjZCO2}h6ZrhvkpFpzM7J?BOOYr)kJN!v;TCYZ#*Ia{2`MY+RN z2Zv)XUpZk?(%kj!<>6tX?C5_x3yt{pIEOj7!>=Sry4AgD>7S*#$({IK$S?V3TeiRp zF1wDfI{iQM?hSmm9~~!Sc;c#=Xhu74YCK9~Zqr?@mF#Y!qafb~MMiFRcd+0-UPe}I zzN_MP+wVIj`?#F+?Aji_&kCOF+ReQ<6q75WK|vI5ry;OC{BjwM=!n7gE2-^IHHrA- zQktvxZhr4#e#@@@SQDlFz_wdfjdOd; zYvUhxL5ajSQT1!pow_Jfi_q|l?Vmx=_tJ*VHcA4@n{WdLlVV9`3x}@5#C1kojLcLVmvOK@7u&D!T7Og%Q+JOt+Ie14jB z6b9hjTgj^TdSq01)^+Q5$N4+L&wPO5JKJQ2S3(C8_N}p{hG5xsX2tMGYgEMRz?siq zN$nvtTpZ8#H92;~7y~|}T6(t4xK4~vT7ECKqm1kNK)r!scVN)ZTL)0POO`mWZrCn< z6V_k=g4Bb~utCE4%PEtfZo{zY)pj#kOjxPuULcS8oKY$2JRdH^Jg-J()BI~=cjwvL zDuP$-A}f6?<<)aJyGJ(`bwwmtDPkq2xhY0+@6R(L(mVr6zIrFwFjXOp4aR$f5tk4A z%IfVut2s24J$fq|-5RD2p2WN0wn#pVNLEE2!Wf!_J2$rN*6DOM zL=RG(vTJpsO7+aDfc6ajV%Kk}5*(ejD9)sDYgYl>!RLWe2&Cb5$7^s>mS4^r*WPq3 zc|z4{ofJmQ%-SL_IW%;SwaCqaJ-yDew!n^z4&gZ$lj~iieIJ&Eh|D8Q|;qRBi-KyXmx*>!nbR^gd2Ps`S4Bk-J@)Yb8ul zR!8O#Bmb;?HGFZ#ppbq{ngC70#9@^Pm4&iO11|zGh}POnvWt{FdiG92|4AT8B+zmA z6Y%P){n9S}KX9N*aC=kB53`}AP_^FQGOZ4O*q7Cydrx?xT#KjPR&SJOr;kmisp*~X zi*E2jPAw|Y+SGj+)GNEQ#l}5!!eR0g1Qb$-w<_qn~XfUPIQerM=&sQq7c) zr@EZ(yi90~Y`zL~BVGoM8i=@QQHUU59ib|b?`6oaSS`zF-#<# zn&G=%o)ErI=|g!X;Smv8ZBiQ~ld`c&Ydx_ZT&-;tv%@4`AZlF3X82bcjkp}m1Oe!| z#?5O-tq~E7^E&3DTU^1r%oOHCcH&tDG>9wNU zNud*Q0ncwhzof=t*Mw|e`7SUT_XBT#0MgN)2YR96csw+dC<_uit;MOm0hE)o;jSJF z$1%^m{LC_+H95A+(De;FPzgFi$6w5-%j=!nkzRNocAS^LNWD zeuMM^S%dDN>#ARevu9QWxZA^|{_Uo>*i30OgvIr9(So6NZ8zckcV|=%W+Az8>$>oL z=!W_|tSlc*_cAY|c*6R&Z=JW)oI;+k3gA233mLzu#T%on{IeX1e?|XN@mW)rvuhU@xKF-E7QTV9@dBmu{coDTH`WQ3mp>edrTKjt!?CZC5gyHwkzgLQOxS5`e=j45 z|2?M54UJ(ezKEha&$Kikx}T6)av5)z(vJpn_Dv$5vQw!$c5;*D9nG6|e*|OaN(0L9 zM%w(|*;qbeY3Al{5uX_lRxrH-y(6JIW~>Pwic^&dKPWj4tS}-F;EX@ZU`5tmDYpdQ z0;XBa{^$eAc|z+SAc#vy5>v7SPx7pMW>p03mry`%J^5!ycnN(zCJ`IyYzze>^Ysu zha5cVRsww2d~i4%lh{lDLv9V2V z;VNSlTiq4{;worMeIdZsFMQMxF21rNXd?fDL;_B$?mcySJ&>o$`RR8(3K#?sEESrH-W?Qt_n=RD?hnD>1G_MJLGHQdF2MpMsZXkkVf z7Iibpk?=*uHvVFH&7d^JYuLuv#XpL z{sYxsk$1*itJ?V3!eER*BrL|bN|HOY_zdR)^a0Z`n;4%x7uZt^5Yn7(X%1g@Ozf6Q zjf{wF{wn}Ago5a{&lw*!&+@-fa3qNM*h#Dp8cqMyci;1a+DCuTO>43X-?QB|n-PCe zu_CZn2!9V;aGGsRW+!weKD8Sxs#8H2p06AUQ4-dl!`19Oga+i0rG1IbkFq}th5)omTWlHH#O4ad4nf5tQ3pzgp%xcE zCyF$m5ySAyf_PJVg;fH1@In}eB&b&`MFh%mC$_xZH#~HFwOBzZc2PcttcgmD{}$GQ zkY3TzhIZsHe@g%7#e|%_L`a*Hm2Nevyrh(tU;u*T#P{*$wiX2ITIKjQ0|h_{yVxli zFgHSoJ_ZJo1!_o!?L?7c)5Q*6z+xX#r^H7vwvPQo(oPj;#f6QSU=P#-m0>a$Q^i32 z;Vy`>V(>5s`LViY@y);}cNqj=blwGCtwb#gc>jKMblNXgbF62pEMVdZOW3jqo!*e$ z#-hO)vGs*bG}%F9251>Wx`5Ny*%~|aR$cCYyDY-qrU=QE3eBp(`&4vup8q=B~!DOB0Yi@@uu7M9u z1M+#%UqZ+CRyq=lT0_^;GV-YrA-Id4a}LFy`5k8STcwL~0jYuTB}bauwH~0u{P^~U z;;bRMZ=#Ii13N~jq&mu@xKqE9D(yO{>$pV zkcI$bZF@vVsNl0oPUqW`{;K{TeFL%XQ$_Rc!lhT$!{v7ev5JRXNl92pPht>=h(Bwn z;!cQuEpa8)FYWpe!ggO(1>kt|hQ|62dfz9uO}y?-!&}NL4f^wHfSMdo2H0-aGd(Mk z%`s~744265e~F-|P_js&ibc~prrA)4*?R>KI_68A@!8g{q;p=26(BJ%()jStIsS`! zCs_}A3*|Hv{lfeAmkH5`39bof()>|i=zfEB2OW^__y6^D6iF0<*e%RHH0iI$$_jpb z=%ewHh~-|!fGUTf*lki)tKb$@t8CHHSkLUM_Ucub3XS(ifH%fXf0(2m{LE*T+b{ANc}g!ES}XmblbSn#qWjO| zeY6#33gE+5*o#_ifBEZ6Sl}XB|3Y_}C_1$_sZF6J>wnr;WZY#^UjVZvUFhXM)&1LK_nCD1Bk zeOdw)=_Oc{i%1uEy%OosN6|6q6Ev2j*CxO43mOQN*NLQ-IsUDQ`kRbrLHcO@3sxLr ze}kj7=+rA5OWShP^e%|Ie~0Bb-HvvnkY=a81dgzV;;6q!`JB&!C?B%FXtARH1m< zKAt;10^wii!@sOH!2{(bCuCAS_{`+-;fI*XAtd>WOiySd_Rft>`c`Nii!8R=Q*g_r1vddWoZvYz6C*b!a#vP*H@O;hgFx%nHf zE{C8czrM>$}~PlF{J8qbS*xQu4ljv}-yvhN9* zr5y49!D;<>(MMSxa4@M4;a`G)SEX)=AN)&oPJ`oJ3D~#<=^qw=Y9@M;e`?SB0pK8z ztfnG~dBEW}v?#0uXNguyt$T`zzxKR8#QoKnQYQetNo0}tF={GmqhckR=k{Y-=g265 z|H7Io&y)5Pb69uMe#fF?il`^z30{eZf3z+ee-26jBk8@=)03q}cXefEd8YfE$wC*d zEHqKlY=`^bwAy-yR9kWebJh#lvbIkq&D0!prYKX}lD`-|{9y+9SiTKvBELOb=i z=9j)2)>$+Bwr^_5sFbTiDLLc6JT+siB>y44>B>_bRmS_L`GNn#vHyKbp@b;Q<&0yX z2@Kt30=EigX)0$@>hE2i>zRa~t=cwISvzN2uY3yJbvwJbF+$yi*YJZGmvC!?nU}r& zW*`4ArK9N79>Ezg+&J}h{hLbH;ABa)rY(2SJ%kJbI8He8&^t8g|N1=tuOAo;>b59h z1{?kPvhf!M=Y(wv_1|30Zx||UW}kL0wILrCER?sXU|@WkHi~;s`+g9#H{EDa5X=#r z6CV*Wr^~*yjmDPgN8IdCff5JadM=Q;I2oY$qxbS}Z1(^52mkX-5^*5e45i9ae)z&6 z_6nBmlL9J}is(Z@s)*BfUK*%&JpF_0MA_%XIZ4eS}M%~!-_JED(Qi1LQPL-z9qxH&uqgR|CRp>6HZOf4*Kt!47GRR=g(01CTmNp z%}*^~Ktl#D)_*YvEt*uXB-2>$ntT&A2!VWbW8YO}%chZ<5TV{rl_%h#L5#`n#M!+! zKY9L390*ad?lGrI2biH18|$vkEjc6s>$Xz^g%3(r|2T^O{p9}T%|6P|I-rKRIUVe? z5U~O?WPJ4}a(yBVB3Otlj8Vb!nfxI8EUwrjoF~d9z47m?Kh*?K+%MEUs|h@1UWgp3AIRQVj5xNG7oh1b^>LN<$s2QYKPO~T&RJ~nE1}*4rjyTr z-S7DUPO8B2sWc(CueO`RxiZ^@d2x!x13*9|KR^={y^|I6WMJ2e5pZXl%uayzv#-W} zZp1`Wb^S%0ovTI_sQvyc1AF$IY{7Hj-deuUX5oHeY5jPO5NErNZL^(eO1HYMD_-Qj z+W*hTSGNR1s@`A#+|k%Heg}Qa8&)b{Cjzh+h!v$G&~dgp8bD+wx_>w-tae$GD+b_t zqZY3ai)5hy-D^k4f{TNj#5ATqQzZEjt2VP&0|#Q^@vkcBkh-59|G1l+?_dRyj^)~J znDxlor5pm%R1Oexr`l2lDSod}R%mB310;aOEc`h`Du6Wla?03`EdXF~RRgR# zZOd44K9J&Dfc!-5u{Zsp(S6%&3c&Mlo*TM%0?d@H#@#Z_0(44egjgyO9g~kHQnc$J zY;xYWN4gEJ765x)_eV6qWzq!LSlai%FhMQ=t>Q1LHT0O!<_+}!py3KILbiTB%T_LK zR&JXyyw#|&yaPU{ZDwavE+vcR#>r(W%}UEi;cEuQ@Q!-HwHTgpvA}CWl|R>;+1qa! z;?xg|inU8Ld~~#;j-F9>is1RvfYK$eO4W0vMP>6H7TvYo1$qgx^i)+#oh+~C+R)@G=@I-2Ow2;){p|9j|6UnkQtA^K_zYkJzWID6rL!45lm0-yyIHBY zwwCtOF2oW$a1{R^i~_5CFNaAl69DgIi&!Icqso$oNc5A!hk=d|$QV$~Wn zDY4sfJeK|Hp>4qUfM8m!1jaUQ} zF0vpt{c2~ebD)Vlx8{1e;N@kg^cy9{Uu=*L1^g~9xZy%7*i-{zhZf#Zh#n*b>URb= zlZ7Ooj4E-A6cuEV|DLQrg=V^!C%TYUoubu&pO3hXu85xod{ljRa2&_T!Fr!{`5%() zi}E=waw2zH-tRO-9%#6fThF)R>N2~9HUdPRkt$Lmf)F?RQ}bqDe^DgH$4t5+{D1~R z`hpiyO@QraQ1wYct`{TLlgrjN#8?Sm%K3Vv~i8m#@2j&HW&6M)gz> z&cCTus>+r{t;nNx_|jRs2!q7oS5%8a?4h^9v&3i-2F^_dl55=kxsIizL$+mcO7}0yrfmMlyFSRKyTnF;gk? z7e7*aO%t>2g@ETrW0Siqe!C|F^p?Z`*yks}(Z~T?<1qgc@p#EYTuCn+w-t@a>$~fINiKL*xCCz}Iftd1pU8xN3&r z2LQz^JC`be+Ubk3&IFRpO+YcFho?*YxpU)_`>+X!=wIi0Lv!~wvQzh8F1nMQTFV#P zoc#WI25cxRwq4%21D2@0@<@E5Eshcf21yq&E-ady*z?#HzOv#bdwi#5WJ0|32N1{o z#_C%SucTS|U_57)k-L%fbey`4%vOnoKv>=Yar(a0IY(PulIZE%yxS7F_?$?AMWbRP zK@!J#NmNRwa%TQ)%^Y25&u^Z%=mq^Zm=@ZFWYb2^!b8@**@Dy6=>9oHKoYwEg5qH? zrFk;GTLcCn;+6@1&WrSpDM8-V{~YjkGf~@^XPpyO{o2oYcn`4Ev|OCOq3eFW5&7NE zGxQXMBh?;swl%_UPIw@#!hLH&o$m7+0#-mp0$*9f$Rs+T04Y1snQ7)`pmLw&_frGU z;zPX0S16*~+;2W^nw8#rJQ;t9yhsSn=d&IT1{#22DYEV70Q&8I>k(sVM*;svYxwFc zbG6L5yyb!(6#W^Wi1U#Un7ICMdw5BF@RB93+L$N1&c|@{o~xGt#WNdF`i`oEf^UF1 zhpMGaXT~_OAHa~{lh^#=?Nf$sI5;%ohDKpIN72u+azqa5yFN_WTQ$b`$RlgIBgw{F zWe4aa;+i&s(ALn&PjNy!+}-G24Dml$J@*23?uJO}yUa}a^D-fY`H~t-327@isav`+ z*iy^8YZ3V47Y2oO|oZK%GpJ8o(QWf1=b#3noEw z=7TR>USpswY)4Nz#O-S}(x_Gkj2!S>V+Edy&fZ5G@*H!=zfSUTq<*F!zfCTz^91q> zS7eWKJr|>lsuE3l*?BYbi|$WRo$Rsg#_MLy>jRbiA!`Fb^Q%K80qC-**g-muV8h$z z#I$9HkC+>j@i25e9a$Mp@7>Zmx$hz|Kzo~WL&*#WcJ8k<=OK1^=G0hilI74SieVE) z3#JOG#>aLvX-zoU{Ih1yhVEA8rn7oLvn-B|m|6L%%($iT+g`Ah@CE|!(2)z$B|eBY zW{DO>fH3Xj{cvz3IlrFs7`WE&WxY}`1@4ukya>Mh(-|D9BOoS0?O>DUo@IH9uM0E! zS>{`=@Jn&6;W(K_gPv@@<3J2Ek#R}&pX?b(qvZ{i7X~#hv2X@<_`(cMD;Z6IAni+8 z`Y7~YC&mYbwjQ|rlU(gE1Plb`on_|zh5?}b3{^;aMS1IuVrEo}NiB;D6_Ku3nQed? zFhJp8!t;d3^D`v~$tEt?lk}Zs8j>XErAG=bkJv!dr>c$AQ{af{Mc5c~8bUhh$w~BS zzZU_if2SM*z{~@dSf3GN?ucFmEQg?Pb(5NC=|p(_BJ;-ruJfl zyq0rPtP?8?#NIgNjX(Feeq54uCQ0k?SAG^FnXj?il6c$x7E6Q>{LN{-FH!M!15nW9 z*poc%j6m_%3Ztv=IkikA#9Eg^r+NuQ3EYp6$PhF9J79LTFXkS&4h!z??%p!B8?~zl zEWBK30xm|8lOGJ*fFSjCr~+s*XTNY8K=U$LV|U;Qp(Gt~wE17V1_-T^O20Zob&+i6 z>=2`^M-kqy9(xR}7^)ebgtCMI3{Zn^iV^Qz(dtL$x6ZKw`y+BU^hbl+=BSa^g6!~o z77+a90#J;4BLEccKB29PkHpfe1dJn5(vbm$0=a%OR?iPw6^<&-cH*9M4d$A!{X|`7;K4Y#11Y;jIrw@^-0y4dy^M+m5O`pkrh;YScP@Se-GizfJG= zlSt+3Ux<4%s9A}Jjb(C%)vWF96C9%GqK}qz_D!W*IH6zggCA;$^2c*>EX28zI35Z<&Zr1Nm%?C68_k>v+S4_)xdLFK zbL<`cyUa>Y1x zg}u2tr|@UE%r_lh$jIcNcedT4L^z~YZ>c~r?bbbYeEmAN7@Vsw@I#+anKaA&Aq4?-8AQGaR=afdU~-cciD|Rep&$!9Krk;MNY`AbrSV1s=++f=<=!Tc zjt%5Cw_;UZ-^Msz^EcovHi_Cmn7rOop`7y60cWr0iZvJy?$Fl{jgD8C z%5*ki9a?@b!cpwb-J-5$Gu;5B{+ULU*hK9-Skme@YN!pp&1J~V0(+*C$%d{NbO#%N zMuNDU&;c=C^)QG&Ii0jvG{C_N%3$x0`ySL;QOl%*5vM zeMBhSn201>R|IWP+9%(vp&&j!I;<_wLxS za(K{8Dv{55SrUe7MPm0bJLSVwBN!QNH6yU-otKHXUS*NR2VkFJCTS#Ccksj3+rh@= zP^>-C=9N0gTdU-5Y!nz9p*FRpDr6vy573;Ej}B8Oz}!@tr{}JRJs&FShGv`^y0O~2 zX~ankp6lHu%1Ippf3(h*{;5`z~QsfdpWc$63D*fLDA8EN@zWs^T$S1~h$&|(}ngy!b4sSNJy(1zG33xcPDfFzrDE2JAK4fVF zAdr$Zl^ZSHvl~0DFSLsxds#~RrXE6Y=%ya7ABxM~h(}<#NiA9hq=Dlzr5jDZblBTn z1*no-Zy@=Q`SEIZmd|njG*VwrdkLEYz_Pukzj&~>cycQKt$9pM!l)+XNod?f<@%>Q zhmA`N9%8Dgvsc{vlNLa+JJ_S0GB28vr%Qy6HuV~)Ta`V~_i@hk=ya1{!B~$7KiA;j zA;hJVza}9#I)EZ7jo*6AEW4!K1JhY8NDEQ7dKfdWC3xm01lj_lwuSEk^Zj%)Xe%eGfRt6*C$xaO@K9O>) zjKqFoR;l~^Ry?5Xmel+J2w9KW2oKQ$@UC>zk_*QAO^yPTJDU*dD;Pf-MFP57G_^jv zKjz|oNAPwD^N6Iv^;~fsOz!lLAI-&*AV0_zuiGAON)|YDh$^NL=n`B#L_SfSHgehX zn${5>I$y*uz$o!t`cn`~ZKQNsx#(TAOVv=P!DDZQ|1VD?AERb3*E{R@|Meo(*Fnu8x#)F6RU)E~_}X!c%{8 z*t#j)iR-{F>N%#T0QY8EC#>Q(4@O!GGbswJk+uviC-dtzFnICIT2+SUB_Y;a0uw!N zgSrIP^;^`;8`m6@D}kvqKpOqLlqz75gHBt#y2G2Nl$U2^5@RyS+2Xe>xqxAmY)+AE z#N9B`RGLxSFTp@&(HO2I*pJ!HGBYEa8-15QmX#E&4nM7$*0mgk zVes2`hCbxIJ~hvq%)YETejR`IpBN%kQlB1y^nS_zG}e~lqIK9XUiRoHmkkDGvp(3p zKj?1Fka(xiyCSpa==~>Hmil7b8SBTpU_~XWGTEARe}B^NG8N-t-E>Q@3BPy7F3ATbr@I~&rP@qig+pX2&_WVpJj>tR z^Rs3FUKT~_$t)6kU`7TK%&O0Gx0qZ46$kZc#_CRzxKx);#ym$CZ4#@%3t_dsRQl<*-e zHpa+(k|LZ#m2s)f1oLy{CKBjF!KnTy87q_EOVd*1c<9y9%8xR$^W1*1%X!VreCA#-9n54ELn?M+f!pR7^vR3Ke^Mgiv&H=N@pqJ&4Tyi|`^%=_=s;(QM)8l=s{ z;dLSBRIATAKVteb3Zy3Od)NNJ>m-lA#JhrcJGIl}-?4szs(dZI_z>rX#%h9(Oa^K9 zBU-F0ylwk)U+&6<*_xcK&AOPGDdJUIdaO4B-0?g4R}vMYon*k`o75_7nFT6gclEVl zdre^)WYVLS<>%Ok2F-1NE?Nyvmi=C2<<0F5^=YH1+dh&6(d`RJL1@L0WFdn$QLx9> z?%oVSnXz83l#gy$9iO3_X4T=n^(`@67NS~Y-K=w&K`vt zckt^%s87o0%KVbav+*{V)KnyUaHx~ti~lx5f?)4%m?%2R`_ya+3)W+uUrL(m5GAbQ zy4D8J`>$-J*2EtwzU^9k^ZQmP+52fuAeM~b6P_*ghotI6=-ljNf{cOwn40MKG>^sT zNfTWYsHs2u|5fO{8`_}LfmYz@ku#{k^sm$_5oB$L#Swe$z76ILtjuOrAqi{$1}w)Z zUT9Bw^7-{jI57yTMJv#|0s6JJ&q?3jTMwI?VSX{5``h@*w~MY;sw9MV{6#*fgZak> zJkR6~#Xu6Q**#wIojWY zp(HO&H8O~Rsq>RFk)INnurCbscVN)Wxz~K^m5q+Qee=Od?>3$WNe#oMV|`ETUb+8` zLi2;8r}bwJ?>_Y^Is_9vT$2Kl5$SLW`$d^yjn3X*l(O!ugf} zeD2HU<&f^H(k6X5H$?^Ao0I89pG%v{dxT1Us1VnaR8FlJbDI`>tU=j-b4ZL|x#s26 zcj!Xp;Gfx;d4t*#Fzl!sRGVhhPf0J zY8Kp@$8XQ{J2T(8Xcy=|670?S#)ejXZT%nvYjdG-BbGbTyAys0tTyL;TL=z)#XZrANAGDP z*a&jp{7g-Blep(h@y=g-IS1eBy+$jIrlZ3(>wXoXd_&F5`(V6B58jdx2W!}Mvr5!2 z>~xdcS0HC!>%RhAWjW&;uY@KWSE*p@d55j4H#!XbV`h#L{Dpb)b;%```wB{5w$P=sZRlb-*Q^UGB01~Bjaw8qQ0|GOZQ+WrYox$7?tWU#83>#{ zK6!yvTgyg0|EUOJ?jPZIvtsA7-ocJa>YYv1&^r$I{iCd#1j*gAo%h#T;cxRERmR{9 zzA4%5HV$ke+4V;}4Ost5&0qVfkIpuEJvyCHYYtmdDZt>XCT7iPq;{e29Gy+u8=A>v zd)D^|VN_e1%`0T#f%d~29!HG*#Xivlvk^W90WSpg+}$b>YAi%Bn}}*D1{M!j!Q7V& z;_=k zoU^pB9U`@Ogmoj2=o+^o6Kk~dc6F6OY0||f+r7~&4$sC)0k!Eyw{usGo?ZIJT?Ei= z?SpCcuP7uv=)igRDfZTVhC8tK=)5h}Zzhl`E{$fYrxKLNBdU z9awF>Y=PuMy>C`xy>n4-(*p90+vuPs;ukUq^Y;7u!x%ZlC{iasI-IY`6 zIn>!~Xj20qo3X^F8Fz0J?I8l%jiifjB@xu0K`R|ndt#}F>BvIY3g~NjBNevzRQgGq zV$K8L+4_^-hG|oyhc%OiXOWmawG)x( zw3W;A;Yn<=Z8lvs+_isgJXb#Tsbpn|Yd4K!ut2qFcMuV0koGV^gi@-TZ#SbvQoG)! z=;oJKvHFfJIFl0BdPfHe)qWn~U;yk42ZMuZR!^5Q zR5M@>faF}VIH+zsgJ2$|>Nj^HlTE~i%dB= ztxGAYJ!b-dzM`KX7fHguCS+bQozpE=2d``YkxS#eIEx62+DHi<*QdjcOTFCB-TA)y zjjenwNVqiH-MaB$Y*n+9#ghF9>F6C)zrC3dtJt)RRN0$U+vOe6 zCR_9oaeMNWxh*Zq%yso+X4Z%3v8;RiI;eTr1`I1-x@49H^!H~aNED0 zYoNXw=e-Rlcr)Ylye$o@2>l)S>w(PcV}4iTmxJ>0Rw8T%1`@F);LE7_ZI;GK1Aq)uu+ z!NlL+M+BjIbHGtI2YvKL!RZw7dq!$qw2aUC2hD+!HHR5vPb$=&ww0?;H|b94Ja~k! zsfKzvJG3Wq{~nu=cE_j^hPmJ;gYM&(k@`&RFUI#6s#IrAY7k?$Rt+p=0%D(R140_v z04a=$F*tIfpzC}3Dp3JDw(NI>c;eKd({h_KMF zQQ~HZ_Eb?rl@olQ1fxjtmS%=%olR{AY_peA>>u|i00@dp)`wfH!&$POsGvrFM zxm!0)Io`p$tz#eC!hD*Je*{|1C^96_00K7~SYZB@K~V+n`1alwA!Ymx#% z`-w+3kx?+Vz233pBOy60&*l*=Pa1){&3Z`q8{T8h>hvZTaX+>;(W457(G6M;d}%G_ zriAga@gkm>JzkZ~r0XeYq7KvDdx#T>NP;#-Jt>!AqSxyy#2k(?STmNLX)U~cbWpyR zF(J8w70nm8{ubgnBLpe13hdKml`!1@oDmZ^9(i zQatF9V(Fy8nD=eY++Ymoxflc&nMfZ(dQapJFXL7BBn_VHutmslXH-4fn=L>glbge^-K;`IE(Mj>TneT-xOLPCugzve z2&7W*a=Sc6!(1($xHS|_^ihdlSHsYaX?Dl3%a?b@WTYqx9W?LASA!$;}$2vYtJ+-7{s;~6KE~hxajoE5VsJ9IE zNA{T|M!i@P7;q-p?;qDlftN2wyk4WE*~fe~{XHr~318}krA|EIKCMEr+idk3r+Ll>0zQ)#)so`t>5u#z+x#tEzuoJ{#@eLQKSA^g6-l_OQy?q}n>K%Fir+(3t%M zwND8{ytbz%Q%)lU|0E7;nDFc;*0Af5(EUIyzV%k2$8W-5q1ztYZtbU7&!#LYg6*DB zJ`*aBz%aS#w^VK49A#%Fsk4r6c9;w;6g>agDmPIO%C=3b%9acGe_Jj#`PhnI4M&`6 zCOwm6kF%evta)<(_uT?asb#1LO#?hCG2(Q#-0>I);%#f&PH>+ksiW(KMmfQi5}98H z$m;H$v=r$Oc4|A%`z{0Jc8-m)C07=-;hc%sC!F{Y;?CWjswpdUAG^b)TG25y-Iq+b z zMdfus^4FK`#gOc1vyaoeK` z=becD^<F|E?P(W5xURUt2_O{W_Z*#-jl5%G1YsCP=1UG;D0uJI zm~Z76{JG*3Zr)=2agd3+LQ{ZMQ;wx_HSEdMIoXb#^161Vb(rUs&u8MF2V;&2 z)bvLuK-t*tp&N17#!P*+-B!-_J0K0jt2$`AM@yKeNu?OM!F8j=7X2Xk!X?j}rRrP8 zBBC%MA>n`_cHR9xP5!teAs^k3{8u8Lp77+d^~y9pgqoqZ6^qQfVyvxA&4bx61BHx( z*_Ul;DO0))ZLz|vD?CX!xJbeRsw5e`kQLAD_DU%aHbVaGDx8uoj7U0p$H)|Wq4J>q z+((6NdzRAM7g7f|dmLt7j?+MU}yQm)k}$MIypTp|rtk zGQ;c4_1b|6mRed{Mj$N+)Pq(cT;Bl<(;r^Ii@6H;_F zrwqo%wL&amvR^Gr@X{iS!(*hv3;N84bK*VE7s!#h+$P-dI%4*i5lsz^UcE~@sw}0^IbI?iPwQF3s-LhgY1UF#U}M>O>Vie1L)k*106`6o zW@&mqv?cK(q)>i)vZJ1)n}OD@N6M*83L->);bHB=%#};A$Kgmg`D)00DU=o=H6VW(8 zTZIqZsE5zbCE>3NY4q<+Lb0CRn>17&ZZ_c6!Y30 zbsuyCrXHOT>H5TsKzdwSj{*zt&;TXy60Iqxy?He(&TCSy-xtey*X{`}YGq7^7@qY8TSlrZidh$Cb>G3r==O1k z@a5pN)0{^wnm0~kSc92Ux>veXHAJ@Coja1tG{vlx-2dL6ll4x-26cAh!`NO}{V}-V z;Q5xdXU{2vjL*>ew)wl~=Wp0_U!5pj<0j=?KH{c&974ut=PnPci%~HRzMSotna32k zvz3cZ8b!*U>-VcYlmWV3b9OC}s6Sv0o%v;+SC_0GW1fZ?8#XnSk2OSek_6%hwTW6u zIRR`B)or6PGwr0XIgXnP)cjv82t33cV3ae9H*K!FTHE)#Ea^6c1X`AeXkMU?S}}BTJC?g%JrIS?e>x_Qlj0ezZo>wi=VPl@ zqM-EnYO=$5%@6uSW(2?K*VR~`W`^)}KL!Gw$dws_V@F>r_C*fv)Pu~)JfgrR3t_y; zSY`9X)B`ZRLh2>E`n# zwmq8Q7KiO)_B=pORGY1r(FsLvg0g9N0SvOJ3qYE;rH8UIgfG_Qt(3xtPHi%rb1EPI zI9Z?9u)=DQmmxsAAQ=Aw)NF&@_Xu?fS=D@ehXWK{gxSrj~e+QGh-jb4b`P)vdM7 zMk7>V-A5sJWV{xF%(OiiEIE8^|Mi8UN5a;8wt|aLaFy@w>xmGwNwS;XMKAndF6{Y-r|vc5P1W^);1GB zz*Lz?Q(-dT2ulKzxU5V9x@yizTyfc6Q>ho5CW=G=9#9+4Wu|1m>9pvmGItvnQ^xbC ze$D|_d~*DjN!-cN9Ff!lC8f!dW*+WCyvNU%0z1?*W+b<$3OyeDy#J|YcYp~a zF(IKBK>F0x=;uAi;3cP0Eyu@mUh;TW=-F)7`6Rtb$* zhW4ZlXVTvrf0#ErEby@GIe6^)^YdM*oAH1!jO#Q}y7e6wj%mrQ1oNdY{PqJ5OBL|7 z*kZxqN=2cjrCDWcr&!c+0uWE^xinxy`J!9n1{$}8@^P-o~*@YzqUDa-;jP4x^|d8FoT zYpQ_H7POCF{Na7Pi0RVcWre%c@G$m|UBov=v+CadGz3+m9EDWQq&I|YS~qRhJ9_GW z!^hE-RwNTO5$wkP0nudao|^W#e_EuxtX5sDD)3 zHsTtno5l5fG3?d3e%DlK(L7w~1pt-2jg;V`)XH!)-TiI$wi4GG-|Mw~faZkO2FFBv zt-nC;IT*0@B`DzU%|&|-(z}f^Iv} z!0vhJesq*2iB3EfX)6&Kw@4|oy{}N4xiSt=@z(ZNzPMEtQ5%FY59Y9Y(6&RJ>wZ4( z&D_W=6JY}>*Nw@5=+oB#UzbmkE4W$OI$a@c*sxCSax=ts8gFUyu3uqyoBAw;5rP@^ zV&<79wzl}w$#tDH6YuG1H*8|t+9DfhFv8DM{xTtZM(u1eW`md7Hy^{OVbxlw;e*F> zt$QLA6mRrgUD_NOIeE<5h3t287}UQs@VA@j&dkFg%@AR7ty1SHf5$Clk}dKEX_#;v z=V2!QaJBUueSdLoL>Xye5UEP%H|!;EW5%KWM?D&L%QAqZTZib7GQJFZx)tq zf;?8ImXKr6!jtAR$i^3M<4MFVjzfJigv1sCmQb_1>59f$vSli^G|^U!9+Ufe%I3GB zseajZMy`t8yo)A}s)~4m73|GGX2a+#BL2KEaLdBOiCDRIO9awO&q)9~R$deoEG=zn^5O&O7kmj_Fm6ykwB`6?)m z-#;M+u@qGE10F^fzzZI&o3QBKk2wJR9O7otm*79vI9(68Ue#%>O+08lX~&usGjpFj z9>aQMQrHnjpG;~xz$j(mLB{_*g@BMO@51rMQz_V8O-|_;X0*r}4}SX8gNq+MA=X=; z%-S_Mp_^$gazsL(uvjGyCQzivlPX4!-jjv}4o!Z}F60{PrVOZv34+*MrX_718kbw9 z-YR^_TsOCAG|7B#Y$`Z#_{g(XY;N{)ySFa3mjMn==fOX=G(C8Cc5ZCM(@P1PkK{Sq zw=+KAP+KF<-f^I_+8)b=VqF@ zN{Xt;Nq&aq@n+!O>%HFJ?lYExwVITNX&y^T%@$U7qUbmz*6tb^uwLF>yi5qS)AUtT ziE+3cmtUxN{(}%=+394tPgV<140|zVECZQ!3*$7_IVDd=HM)UtLq)}mQLmS}SwEkym}0ZRTd%D_bKA7%`=yRXt@aefXk z!6;(vgt!gazm@5Ep1JqLjFQ8{NW*@-u0ZZbnzp}(a!+7M1SDNaK9T0Tpia2Rj3-J@ z#7eaVit+I3?`o`G`@@CUeKnKz!?MAu7|^A`pgH#ncm*!pfx;=zcm2ppIdb>(X%Tzc zg#4)_SG~qqKtiJtL4got)=~())tzY3k9$8cnN3IOcs&%lbc=3=lh^;=zY|e7c?_s( z?9lU5K3l5n_}y|W8Qfaj=rh4SRJE(>IPMRpWla`a9byl0Z2f7S%m#$78WN+6Pe-4r zC#j-7;=i}n9d?AHr`E9DGyzQ%v+;#e69a&+w26vSQm}Zt39E%?v70o%z zWle^#RwE$gS=0Ax+D|d-+^>scwsCSg$GUHfy3thv#n;Q zcR8eRp4B==DXDiCD@9k1@^!?FXcZqdA^Bx-6t6V#=p&F9Qqy>dN$mY`aOiFm>$a4| zm$YL$ZD5(HT1Es{Bh*q}?Ks+Cc)3!iN#04ArlNWd6O41X-=%C<5mDLyBG>Y6VtesS zuiuiMC_%Aa-4|vTq*=up$_Fw1dAYn{jH6br&1y_kbUL9hyC*tMvn+H$H~u(+T-UU- zXwp89n`62@G|yyybszEpzgw%X=DH3LulVfNqT=i3nBQn6V>uN0fWjvdL}nA{hNgNB$v31EbM=2ga$ysEx*wGr5OdzFD|OqD6-{ z1@A(_)eD5bff7n;$=q1CU1v2&Nlh|(8BP6UOtSQGfRuGL-CZ0Jxa!~G)ff7dsJ!uf z`e?^u>W&u;n*dC$FDcxPL-0`e`CkU`M?oYk^c_Bi?HqHs!$o!T#g@AyS_!HbD263o zGg?9QfR83J-$B0po{+jhqD(`|k82T-9ad!GRk0wuZtYPD;<3z~$xFqlV{jiTs zgeS=olVm9MN_8j438^bd8b^(XoFv%Od~`|w@&5k!`R`~Ahm)}WzRVDl-Z*_Z;R}0m z<9^VfWty5iNs)FSinv??C&d|JFT^0DrlKtBG~Q^I{0zutYk^cnvFPoZGod-|UaWVM%N?MlAxjT;YNNQ%BtIYD#hK*mCs z_^p@mr;kP|Y+HVH<^T5Tlqd{Wx*i9r!vG$3z2(xICt6mqXNm_0giL(IMCISAjC28A zCFYszl^L}6-pY@BF;05`o{fY`K{?>X2@)xMsKmK9tz7=Jj#kx2)jh zg4-217juD>>RU4eJ;zYhJ?6N#_3=(U2fTEn4y#D$@J;LiG`c^|9)HG1*QV!;dd<4+ zvJ|b}>xW5F9_0<%GPpn|@Ka4g46ol27dKG1f<87iUZj)n zH$O{w7CJp%sC5h(i`L+_^K;p_!r;8#bckfu1zy zP^ZXTU7f_Qod)aRF5CkZr~mypDI#-C9(}6nI!oBUq6sumxj#M{(e$3Wu<~|U8KLU) zi)Jck9r06T>Z4mI=1RSszcakCiYFVoO$M!VzevnkgEf{9*(i|T;wgk!s(C&g28fmj z$nm!e=t-sLvZeC?;J~`*e=ck-04DkG9^{vAPtZoP3>5mdPWyEF9G}TQnAGIs~YKT-Z(<$cKQYL^De#lP&^}Jm@Ai~pBa+#xh|P2I5_9?rA0sz$30U_RHDc( zqacS>YmSXA>lw{RNMm=QUfHzH_(33Hwz=p4RS2g~aRa5QChni-)c>%RKb9FVn?>9_ zNLyOMq(_NIAC{+UORvBc7fkg~*J{HDhaqvS|Lj{KX3ICro3{#;oc69Ro1uGlo)*N! z3dkxirf!B`ZuB=F6=+a7vJ1@C8d`BU7t_0Pwm+K0`{g8z8IEyvxLRnhe16r7Ch*A^{Z<=D^mAA*ff7%TqcQX<-GN}THiu#;ZWniz z!^nhl)w3h9MEE>ut!m0My85k@&wrVettb(l`0ozHl|>WlgyZonY^R>kLFLfIY&JO_ zrn&Q2h@eSTSE0_Z9Gm_{s-tp=P|ltsf1FkQW$eU$+Jl@PxmO!kS^eITC9HkTjLb$&t;t}Nb}G2^!I}l;eVnK8}K=6V|c_+Ig=dq z|1yJL9vE0eAom@+MA=`m=6`>4@U6mq6nvlTuTTCRIQ*YWU_I(az-PY3w_GHDB71)M z)&KmMjRTNj_gRL1K>_{e1%FyanjBi%qrzcyiHiU7CjS42{ny<8f8foEC?`0Q13Yxj zM+Vg)uD0+yzuJR?Bt>M`h;h9c*WdX9950p0rs_F3iRPaM@Ru#<7fTf(#iP%AW%k_=}-`CB8+h zm5IClm#_bwqW_Y3a>>@KFdk z@ZyT}C@1ft+#A97V%4b=-9kIc6mk6J<^J!F+*07Lf^{Dy+y;L|ivmG&eZjseS?c?X zt^DPqmn+35VxNPz(ctA!d^JEN_dGAVj_Gi0o|b2F=3Le6_A*Lf_e^GWE?1QO!UTc(lPatza~2V`U?A` z)zl3@6C}>y65Xwmk+ku`S0@8<{;RDBoeF5MvQU2#?=UD=d9r6pgz|s`y?osJe}1gM zYtvwVd5y^dk2RDn-FLjR&<9$dNV#n?gLybE&W@bMJndHcxs|Y_DjNZ+ppU)dv0(_` zdLF$8ucOK#Xy09AzTLVD`WlSxU_AQHjJeX6xPtGe#4~H#!y+Ezdo7-BW@>t`q7?fd z>;=b^2ds%caS3sNk{CmiiF-;Z&EO2`4~bNv0z>xOvx4Szh5N{w1*|=Qu&aFvd1%#q zaR>z&1*ArIZF{@;aFN!t6*84I(6r@9Sl#weEAG0i))RegvcdT9bb+5hXGH4SPYfp6 zb~A{dH2$y#q%-1cx}X9cs~Y1|PJR9v8;b%QX*ryg$_4`06A!$L=zAJ)e&>`fuD zov29A-4v?rxLlz5(7K#)UkR)2dv`QbA7Blb599)v4vUcNH~1%`a>~k)yz}qzdOm~)hf$eT2C=L!r!7~>=F=fmPdKiZeSh@GpQB+=G}ZyWG!~K`G3Sz z*ITN{#sd-dZ?1(O(ZCKtgL+L`p%OW5)V<3Z+^N(bIEEX=FI#jnu8l?z1sI!PR_p`= zbYe4|;Z!rkK8>yi9H6E{Z#qF=U8t$IGvhSJ-OcZQ{IkI_>h>b&UAMkQDrF+jz$|%I zfRxyZto*HiSRN`NU=Qpr%QgawSnAeG-=qW_#{1G{5m7z>EzLqBtv319;AzpQff`DY z^IOmoOnfo)8k8w}R|hkPsU6IW{9WdK33KG+U;cwF_s z=5>~(5jTv(8pzAab>-SQ1CP@Oe4axEfX6QiW@k&sCY&Mb2iiDA} zv&cU%@Ax_2n&}AeMkLF=gt|`w0hivaJJ^L&uCW04&0q);XOsWh!ogOK9*J53RB=FB z;`bFK;01cTh_oSY#1?;Z48Z}@GJO|3TOxHS@ON>b*Zd2!Zo@YS=HT+qb6Xo zMT%S3t>tuGP}jMQu^S@oo|Mr zM74LLrvb`f85Ag=QAvbwk@7nR4Vg(i415ME8Ol=Y8V;lYeK8qqG+ko@Z4V-XbO^fc z{}wXwdrAm{h~o=TE5gS4n|ZIaLj_LH2FvRX4CEWoX#V9ENGO1xgkNmvB^vj{h?!6k zX4($$g3j{Xr%8Zu9Nro*FD*&2g$unt6MFkV? zvWD*^GD|lB291R4$}0nd2d^uIHDXjv?qdze9C6w$_ZA)(>(nRM4G4}y_)Y~TukkRD zfX|?@d2D}ulu4%LpPZ-&ML2L7qRh9zBC_;-7i9}T6S78S>Z=VrbMAJaOO>9}`RDhS zUA>?)#HteOjSvygdk@ckLvOo8OOv8!{WK_W8=IJ`li{}85t20YC41bf_kWwH{5`yX zeI3wiDrJF4cs=IbL(Pj7;VZKg&r26zH`_+ZADAyrkG8FWWfNAjriTP}0d|(&#jeOc zJb8g|cpu*>&8#aN>mNK%!u9y=y27#l;hNuF ziGygG3zWmb-kXcyzh8{=^iCUv6UsmA4ZknINtA2X_x*`oz~311U$!|B_D;dSJ(m5a rvvU*yW8$yrO#0jN;eVcsE`>|qa#+52aYlOs{F4-u6D@xE#{2&P`Y7RG literal 0 HcmV?d00001 diff --git a/doc/vuepress/src/.vuepress/public/assets/image/espnet.png b/doc/vuepress/src/.vuepress/public/assets/image/espnet.png new file mode 100644 index 0000000000000000000000000000000000000000..bd3a30a6602f47f79f8a55d1f21aa410b4ac0f7a GIT binary patch literal 6401 zcmV+c8UE&pP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGqB>(^xB>_oNB=7(L7?w#yK~#8N?Oh3+ zRb}0N?m2gveSw*QnPHd#hMfQr6w@TgB_%bqviMR{%cV?7DiIAsr4R%y_)N_T*Uwxi z+yX2xP)HC^5MoVHjZU0>tmf^W){-d(L~#`<(ywzUSS>77*Y# zF#Leq=ZD8Ze%pc^L4!ffanuc2pxjvjFk#c8KZVO|5M6yXY(}pDSqWmN!3D0A3 z>rzpza?^qu9}ZRfQ7;eg3ph|+UX0H_S%dPT{qTA{NJ{C6fg{exfZ^xDE;o1S26`pw z$WM15RTQS3L~W6AlP7?KRX*Wq$akA?)KGkA55Cy62Gtd%Dql*^UKl#&E65oz6gmvV zIsC{;)sdH`BS92+GANQ;gBbY1Dg%Wg!$zM5pG2dS;yp74DfY(sW%8%dqA>72}P?tLf|D3fFh>t^V z4;@1?9pd$-GZ1YyFYN^!su3ZSdQnx=h&gv&fh}v^I!Z~_rPf9A-kINsPh`93ItOKA0d6?Sa2S3g4Hbb~cp5%g0F?q&V#FP}^ zP%ZV~Q&o_KHbqrQg?zBW2T#C-Kg|9S_H11z_hT#1f5by~Mn|FeJZ?Pq=ml#-uJ6sL;p!+e^IIQDR{I>i6*b`@Y@wV4;9k%Q_(^e*}K>$|eabKB)_I zX@ZLTn*TzEVj%zdgmHm*d|9InOBURRm9H*R;Ve{;BSYcESKo{~X8i#@Vh!X;s9_U1 zRE`!mHZ-e5DGW&QVU&bt%128+%I{anIF*(cDm@t_?>~gl@NE`V((q^aYnl6J7)($3*R6DT+`uIb-7Ro0mU9c5Z)Bl6vb>WrWJn;EjY+U7wLy{@8kq$g*Q-Z-?0+X=n|=Ev{Q~a3^$Kv=P5P^hml{S}ay8>MTM> zJo`I?-y?$DL=(lcQfA%9Zl?}CJvO*pvFbBYSRsY2D|%=&Q?$Yr!TWSoM!3mM*+qjJ zorm|aYpQpLT~4ZwVxQU85@5-bv+&km=9+mU^ePHr(CCYx>A`ld3clgOyq=0byk{#K zYb>^|1-b$Qv?7zVTj>akOc6ywv;|MtUk$n2JPSY#%pd4#hv_Z^ZB{htyGGpc3{1 z<< zeB3-1rm`%LoFMN~w?Fhp)y>7w5{dGgBvM`U(tXH(Z-IFr+eCFBC^iT?P2yhjz*;K!Hul{(Ap77CTPB>;Zj`SA0&_=;~MGs%WTmnn#FKnA?B z)fYfvwY-#X{~WV!y1+aarZCto#uxM1OV(l7h!Mz6(baer5eiF-TZg6#+enBM}x)aOiZb2X~Pu`;(=#{9OisB3OwjwWfmHDyk#YOn*;u)$HLxd^} z3ZYldKrC3g4Jl3o=g50N&o~hb%{Bt?rF>hF4+V7r{BGtC@bUY9ZR6HmjvWnY+@x!9 z@4Q!$mE^}Lv0AJ5BxFd@98leS*o$J}Hfh22==JG46s&o&yC65o&K6(c`Pbv$(_Tcs zBp=SnaUsrW>Rq`$-+p5qeq43)bo3iKR@DdQ^(SV}JWN!QCY9>-cg>N?&x77#si_>O z6KmxNa3m?-WkZH|r^DvMcYituG4V-l+`7xr`qb@CTzC5{X<4<3wQ3>=;U|yVRkpu*c&|7f!~$ z?VC_rQ7mUo7w#;Gjsi$X?un^)&&Aija~l$!0SwEutAdWnFAUg)HHZE=%pH@QiM(Os zuxHx_G}Twj*`qn~dpk!>8Q<-*{)kcIE>a2_+{>}>)NuN!gxWw_&^=a7<`iXQUTm!B@)*~6~FLbSo+ z>skZYUhG9tt)a$~UisZjtXldEd|r1j&ygJM4wvdJ|NgYOh;`bLoodI>EW4^ZqY2ja zrF8~&mig33uUlk!(UNy`#J(n*{ zNq*8pPZr3ywuMs$d?*pYH2Z8QFDt{#3x1A`tKU@onD`Q!|E2Nx@tl{Co}PgO@y6Z~ z_Xmg}vF6SZsq^i&7f>&?8JT_1OG4eqK2G%l$(cwe zLrDZ3g+?$e(;=-%PDjw-PwS1(qfwGgbPrA(q~q&VUmt>O`PNDHA%KP@(SF9MqZ9M3G*^ zqL0U}JyahX5i5!ePLmAI2t-RZkt^yfP6^H`>I}hIMV%oytEe*sXBBmZ;H;v~5S&%i z8G^HlIzw<)QD+DuRunrwy3qr2T5)>m1xy*4Kf=a|6(uE@MRg6cuP<1vke2~Ms$=}_Fb65c0rQ4%BC`O$Fri|f9)`v9lxS);#Qp-V;a zp}B3r_7V;&o1>I^z(8L+Fv1Cpk~GKxq=+Z+xjF2-tfUTwwoc=~oCLYa+rcqbo z)9*cs%05Goq`B41o`D%Uk~^*HjwU$AZ*P?k2ju}dRiL${6|X%#8*AQPg64)=B?N+k zh>1(Yg;(B)sXv{KxcCIb>IV9yIgp>hxqK>HRKaz(dqj4JXtX{J4fXYS^O;Amdf9XE zw6+|)!GoVI$`$_6Nh$? zVjoSQ%{6JI)>WL`;&;TN;Jj~P<4vN>#CY`Wp<`$-yPBiGywLOXp+w(qF6j zW-6MhTxKC{U}18hHQCUcl!@gJt%4^r2mRCRYTke~fiN225)`hA=Pap)7C#m}a3elh zy;MCkXZG^b5`+com^DXn`1zM&`s1%5Io_%Ea~LQo8ndNXU2uPn9i?8>N?oyd<_*}q z_ML8V<0mod%U5FByjM(aV(3&Br=byTbyU4PI2R1p8yZHxF&D%B{)keGXwuBkSxE%X zQAhHX%5*lAV$x$*iD|iUxY~=7`i}8AGO(!XCO-~TdeJ0?G=JJR)gBn;IkS+?f*?Ac zW7@IdUzm5#H^usagVhEq<(^$!77}

*)hkCb!3@O~IBA-wmESF*gg_{<8v$X5NHW z(ZK-;?Nv?f^}SLQf0Q-&)xJH6h5IpRc>P&4dTdhJ`=ntK8Aon^ zz{X0w+lDpozJi^ft__>p<~7T(;x*wW_hLhYjV3}ViaT>vG#D_X2{LKnw_tCn3x>N; z9>B~`-DYoEhQ!JeRJR1w{5v*K*qZT59T-lcAR5I3$g#1Z5ic(MxhktWLg)xfa11qX zKR+L>tsb>8NQ1O$Sw>f7pC~DA;zeckKGalK`Ws?Cnv-jh0)hDk4NbV~sO zRY!3%kPZ9Y2<-X-S3EFQWwrz#GUQsStLVJzvG(piAw4dDlsH|H=ee*TJX%H3C1PRk zzx4|KH0O@s;V}3K_jHY5rP zQ%4Gz&_(y%m0JE(zd*#kV1M-3weVDp|u+loUjcT*xqTZvdamdqtRc z>Joeq3c>me2hq4kSFnUkM8b_JH9Z>(|F#`NGMzXl$7!yL_e!8O#C=_jwctWG?hREyWb@&C>+qdoUW z;5$S#QzSy@4Bo4&GQv%~UsU-iRhXRyYQi$N9JSkl)}(9|uOnBM>Aq|`a=NbN&e2NI zLRa^WG(`hcw%ri}m&*y4lTnG&p5+9)Dq?hLq7OPxS{IR91Q!L@%llKI)K6yPGP;O{ zK%yno&WQBBmcsiE90{j`3kNi{|Ey}B*ER>R;Xo^z_aDMF(}$TWte*(JMIW=tK2(nGtX0c7r6)?J}kd39Y!R=?qx`|2lf!Xxic z2#&3V#?dU5Lp?S$3AZMPT4IR z)?)rWSF2^J7O(E`BiM3e-subfhVw4I97B5P7}DFJ483i!H8&SzM`fL1M{{Z}vOX2B zGQ2?>JHbPchdd}ZvMn8T1wOvxX(YJ<7?fdGTWoU?G8e{iBm1xtuNCj)ed=2Lh)?T_ z))o);@7O2`?EZd$%x8Bv@%5=c!vQ9hrhxl*WQe@c$@0k=TH&Rt|$=(HCNXPl=dEi+P>!?_oG+Uc$3lu z%Tqe*D3suLaaWpr#fZzWcG}C*S~1u1a*uI)u$w+o@IhR{k94?G^Wd>x6G}R^Z(OBf z`iTU>kisVPnY^+@k%5gl3RvJz4!Pr8zODJt<@gaZoMaUaE4#kId zqrSSVEe}J8B?R-gj=qCN;pY1m;p^Amu9U=(5?w@!!Rsi8mk+8;fLz-yBM|HvBzFVB@E_&aXyN ztE9yW&4~>kyp4aq^*j#k+9LIXM|l{-T;GACFyX2jF?QnBaM%naiy&x7RMN@#(+^%> z00qKrpK#;&na%FP$E%j%gJnz9cHUl(TPcZQE_d)~Ot|_7IQQboa0<5`aW?c9Ps$Qc zVub66qZCE(00+Q0L8GkRP=8~>ZP;LVebC%)$sn=UE=?tQA+y^8NRn0wSH@>1Ye-Kx zHGlhpuX{DkeiYSm9L-SS&?_80!qG#H0*8|?j1J-GN1|{{PasEKEr5=6x0d69U8PhK zx7e;~@T+|>ynf;4_NYIXK{#D5ILS?np7Kji)R7~Nnj{H(UHw6oE)`{g<4F9YB;0p` z+nKg<%8X>^CCZiMX%nPAN)b!oJOs7d@yTrHNp%<{96}E zl46CULxgZDIFcL3syJ|3D-9qvf(+H!-42&nbYcwuc#0X)k7wY=Ra8eXml6TXk_S}} zD+jFV>c0Wx*tuhc#(64@=U7nN!HVF5Vp9l(TelT;7N-~gMgDLW#|H5Kuf4s~=K%qx P00000NkvXXu0mjfidWl&t*wl$1vfChqFV6?k>R{g1c+5#v0e)A-EF)1b250?(X_>&b{Zkr@s1r zbXRxnuH9>`IrkWI%sHY}l%!FR@R1-OAW&pwB-9`vU`andKO?|>yr*KTnnOT9U)YF? ztH_Fr165p{EN$#9ARu_+;xyNsRx~~fM>`o5VGHNi#-n1zRBNj1w-Vt0?1z(0E}}$eEXzke3eT*CIun5T(8mDJ}6; zYBOMhV(JfccD0^SoSs(2`T@!{hk}ESF;kS+YsCt}c zewo&3Q&xGO;B?~ByY#s{yxOBm*lJIhGx8jm1q4>vwaWG~lmFg(z@rMpy6osTpp5fk z7P-pE^ey`HIK+*8tbPt$k8ZJj2rpOpZ9SGk)#BjdJuXxck!{z;{Y@|w2YsVrMl5lk zTqM=fwbD4s_c*S0vXkYEQDILn0Q>km<(ahe#!UZ6lrJFgVEG7ULRY86(rEWBoIc(d z)fF($Sg)}o6fMz_-!F2uP}<+Z$h)~s>Y~qP$R7V$>@CR*u0E;q|&#Ld%VgA~>&oDzE|Vy4kuoM)N{ zVZ%DO>)+KkZmFJBR7c6ncLnsyFLl^(Vt^eY&TM8DUNN>_mLfzW0t9w=^M2B{i=F-krQA6*(2Wh~Q&%GLcF|JGz0C{2ie192ei1#bYkuwYJ;`h-5 zjLG)4<#pb!rR%o*O(&f5&RvT5{pcWDWqw)*M3d<0z85RMzPI3K7I)%}y6YHE($+(s@c-{C9{{o#x|)%h@xLzuy8)Z50v>8!A;FA4$p8C_2pLEOq4z?BFIWYv znfQNq2gU=eQ>-->y-Ml?cF1bua#%Icw`tPCU3BeH*?RWjWHC}ufQiQcpVhe0LDjMc zoz2>gWXb;bI)N}pfOtjBI13U z_=#E*kRo3KlAL%ko5T>q%gC!}vy{<7@!y#yLQNVp^hBe= zDws~N6Atf-y2^unkgik__-DNz=Jqob;;xx~Mjm5-X$u7zY2G$z!fmv);UN7`rM5LY zMoyWC$9GPGdkGEQ_I1da&MP&HX6&LoeGPm1~k z`_zj+D>jsgn1dASpe@55`Ry#CUmwVYSi*(9-wQeVqjaFx+1`n~xWQrY`rnYwd=}J@W=}L30!y+~gPW{yt zo0h5(Y18iRZe?9v5?%oS8E=p@6c3I~h+ZXO*8PFqS;d1zvA)@;I<2_0l-#Q3Ce{rM z29LAVjE#+0t+jcW8Mb>^Eb;OxrVPGix>9Nw^l>@SYc5o;kIx4-Yw5o~iMd}pelDKE zBrl=}UngZtdXbm1lT9k;!pTfa1)&N5vRSrVX4fd&z}RlUH?rk04D^5pQLO~W0FI@h!{zuM$yZO zw7=17qNzK9qBYiG@|G`6Iy6Zocx19c{H?=^kSDN^imMyAoQt&oSa;!OHew82e_W|hbImFZ{6ammwQ70 z+i|*SkDqB|VHB#>4NksUpCBiI@ps#g&5BK$TG1O0r!w#eb}JhE;{y{X&2xw?LKUQ#rpg`TeXWXYIR2{ExZklSOCV^cl(Agv%yz=@ zTYfAR6}ck}K5~-=_mgin`d%Z;px{5XNiRKAQdsN)IC&TR?<{+U2`-^NNsiEKtv14Y zSrMWI|FX$KPj3|S-JKmEIv-#{M9lbZz0h>X2`f6>RjL4rl@ReMh~9^SoUfcW!;Y+io2@O|cMom0GrOX>GBb%F8cqe2k98V~4JFlJ^_pbkA+<#Mi z;Fkq~Za5FAUVDnzvrboT443m?%tkbuR4vnLs66_=UmB2>e*`pz^q>`EGy|Ph?&$R| zw-ca3%wNUd6h-t+EY%29p{?-Yc$q=SyC5$Q)^(5%JoJ1 zA@Q4xLkH|0*{}w0?uNej0Xm-0HE(8 zI7K}NdhoT9B@3i}h5oQ5i=Q@tCCD-`{KM`VlMW{$gPLA1>TVmkmPH-2^FDBLjkiM7T;o&JP zv~Z7%JXv~s8^c7IM*aZ?;6HTpD`|Qa=V6gzy!6LO9`(Z>FEKNZ&S*O=Nu7?!N{a=D5e;1+aewzaX zM{3Q|DvH7a4$qEo`n9MApulgp2I8E^EIf!rD^!v2Q@h>N!p;|c=02C&AVuN~yw7P) zO*8fj9?KP^x8WczKl`azTwHw8sBMg%fiN^vn$uk}!IMDbHc8WAOibecN?~^+DV>=m z^ep;%(9cbHEmt99_lZ{QMIt%ToJ`q4a%xC1aipL*6QGw^IDVxZ1IUM^9p15OERN=& zX}>?{{(xBK5C+xB;x+2c?DztOMqaD zQ=+B(hkt`4^0xtH`_EJK2!jS7os7W`7)(qMP!Njf(DQXG5#SPZ+nH6}voWF;kS0q_ zNvW}Vf93E!8lq19s65;a5;1RvC1ixIZ`8@o1G^wk{vb13vX6HLSrz;E=6R<-u583M zjf=URCRF>ZUmni0rMIemqar6SSvHreMqo=)FvS$3XA{j5u&mJ3)a=uGzls?=nQ7}A z9v(J@6_2bf=!vLRg^x3hj2FzlJzLWsQ~L5+|9tuO{Yf|Z-8tlNqfTF!)v2**vpUZv zM$5sj*N>kpPj*u-pDE-j`5zT@C-^}KtM?8-`G2HqFoiU5_$BiN$(E%tt-tSpAHk0Z%hU^073r52)ke&vj*68{ZJCL*9{key;GkGg13<6@Loy z+c}xX(r#Kkv7nD7vARia$Boi)uP*&ubG6qx>yP-mjMJJ5~nj^@8do zmp+4^OtX2)6i~0{6%D^O{Ljpjs zzzU!4h3{#*T`G0-Gy_%n$Sy%rNP{v+Y^nqI+pO?4C%c}o)vs%{{kIY#;LkYqovSa@ zT&6mZzDLH&a!_^9y=9e^v4VFC(u)@%&|!hSx5o?YX~|aEUiseL%J)wsuLItDe>s~e zlOkhxdG)4A5e7B;B2j7 z_i3G`mQwM=&Du0ML67za=j0I4KYMP#`jEX%SiIisnFsHt0hzP2+V|0t3 zo>(l!E*oMbayk#?bVzSTl7Q=<&)Xx3_$)v3o8O3@6u_Tb?IZ;wT#Li$P>=m$yEyox z53N1QvwEcI+HB3HmbAxNGx&ddbA$Ov<`V>sUC#dt`Zfa1@NX@V7&0Kc#0B28N+FP= zT+BDV;UFr2A_U@aP?X_1(l<0v$g#}3s>J#Jeb2uKm4C-%_in|}0t*YvElMFL3L-J_ z{dJ;V>^Wr_0*2*0^9zdi)1OKIyj6feG8Am|mddlFouu0H@@e)+*%~GKQuI%-K{J^) zl;^t9Aky7NLXPV^-pR;PJtV}!-SK$B!?Sn6Gd2!fbtc;?S)4S#?6g2g*s9#Wx--Er z(m|Ch=8lA+@lHO6n04E%V9$T>T4a>8#)*?Oivmc#x4^la07Li~@XPI&ce(if;l=IX zyVq;uF!F>jj%zxVSuo8LtIhXuzgN!PH+T}$G=eT=BOj~Hu47{lfy;ifTkT+Z zj&5oZnmGSo+=Kv9z`Mg#@%Yt5gtB66t{t-OpqsUYiJa&vw;jrMVt_1dFL8 zcU!v!Q}SFidAaW?*KuQ%3;?4WAc~O-OhFw8IK!$dnN*hvF%A6jFF>K-hLR8{bjpkP z-GQ(oaC6JHgz!1sA<@$1D$m}1$4emzq;~KRMFG(dBw$z|DBf*5OhCs!ze8q(kfJF* z9|J(=@9RA7t8e{>8eJA6;5y`qyn;%@_9n~`dZ#%I7%r!EiiR33~7~`+k-Rbxrok0x*5C`4`p_xE41zrs@yD!>)$2=px zZU!kt>MdLBrM%2iS?#3&a;~$x@%BTM@ywBp7eZ$T}kQfR}NqDgT<3W47J{WvLzFdsB zBS`_}AR3g67>g;6zrqZZ%%z+$Sh=W`g09oXcJ=?Nei2cH=NDz6`*N>sTrP0G@!qf- zah2^K)D=&(vQ$p3Y_ah2rswgT<9YeFW7G_1ZOD~c_{7h?rwTuLAp$?q@j&J-$jxoD z%af#w)CsQ#!SAwf|3n3*ruDXIWeH2$s89N?k*YhjD<3)6Meyp@R^d-3IT@u5aStPX z4AsARz7RbKnHX+HJlS&thHGvxwlyD-7iGIK9R5i$8mxX6eiOar>Q7P`8)Ap#{-&Ds`!d}mNK1wshdpOg?dnYixxmA@Px z241g@js{m7?{>ac?C0Dg=ao#=RY*OurxtmeA_P^2|0^#D5f+7jIi@GFFhvopW#Lm% zMWhb?(pc7@_3vaH5;=dYEa7GQe*;N-icAas6e13+0}0e)W(B|n8G_jrL;m<^q;D3v zQw)gZB40FkCwDTFzSxQ;XmXtLBx~;J+G7sN-lx!*s)0~ozEe>0%cL(K)~N|CBM$O( zD(nw-kJD&1J?Yvuk`mKwaB(S%iD4^_Wp8vc)I23MI&|CDN$t`U=*J{6@Y zdiqBvrA#u!K~&q=@!u!Y#qx`@L&=NOc>&$7_4V)pF8-SlUp?E;R*X=O5-GaoC7cTl zE?|(BSad%(B7R7A|~7YarRra()HZyepzvRUT({;R^e|N#Lla# z(&Qf;eL0n0m-6p1LB0@_xwC=t^;Z+2YXg@CQLYx*jCn`xn7qs%AM;f2KjqNsRv=KB zNWstBgTr!n)*rV(*JG*XGgR#;{%%6PY+GpJkWIqEuJ40k4KSz!@W?gRn z>{bDb_!MtECnq%;?I@O73%-jDozhh%DgEbRlvc4|$FPRu8wUc?nw5zEM#aiX_xmFi|Teuh)-;Fm#Ww1QL<+UpRg#6Ct zc_fQk0BSOXkU6$WU6j_sv^e!e#E13QR4qvf5YP8~dGqk~s&WcJphua1*W6<24=iGW z0-NT=M;ByiNS&IV#P5anJSj1t%Qq0gfMd^NAsGsqjEvSCCA|1A1N8Q9Aa=$pC<~Z1 z!}=DLgy^fgFi)!T%a^-{48ixjCt+zY5x!XXYruuFzDN5zTc99)a*_9OO^USqzpDow z>CLFu%3mAA4hA^KyC-E=uXTnQ_p=5I8~jI4afZ#%vV&q}O{l@plTeD*f*}TSR-&bc zE-eNTN7MNyWfh6bc^aP=?4OB}j^hBo2?4q`O8rxN4Jc}iL1O=2Y8>89|4UDHiN7`C{!_a6Yu1iPP z$U%9If{Vds*A~Itr@lc8S3~{y*$x&aQShZ$^B4*~8;aACn?`dm$)NNED{43JLL+2r zn4U>{FYvijLYk8H`2O{NWO2z#NnPmS08^-ifZ#qaM*PRyIo?1T)F-kc*`Cobzb;j{ zonj=nMl^|{?mgfL#lOms;gON0@~GU`L#z75fKd4TUv83{e4stgld3QVF3C*gP|NBY zi{CwCZkFx!h!(3p=u+aI2WSUe9SioY*vjovAbs|~qGxfjlX+ufE=?#>!Absk~&a)^EE7MtANS#>zfbTuA| zTr#}3g7JrJ&LCPiUC9t}*0_NtV_CR&6&4`{xSRh)f7povYK#v&)o$5x#Uq8LlK*_(ajX!#pV%}4E;k+MX?FEg#4~R z)16^LIV1odm}U6R#rsNr;|$xoz`@&Wk`fp7?E%>c17|B9@EeUzi83E16+%X()3M}f z)o8CCYVP;VZ2pFsCA^0M^b$w%nU>5#a_E`|&XC^Y;IM1E2%vo}^T?tkQ%zifzcnUf z&_KM14Ed_S_E}13+Rt&UVvASHT?_6$ZD*HMZa;0ZKmXu3JgLmF-989J6Bcp9(IKDl z_o5ii&|qfOkwFCrS5eBo6Qql@I?jfyM9^z0`6^WSRv3P``J3rXA#iZ(ZR^rolosB> z$bIi+T4CR zIyt8~m++|*XRq}PKm$Ox5n*8Q=%r!63RKt4MbNnkSvj}j;_<}NW}mC>u|XJ zJg_(Nj)Pjj4Se-_TYZFVC*z7#g>`8$3|R7qXdo?So>6ij_h$c350)9mgl&x}IttcW zSZM~`zCU%o-43ZL)6rUwYnJGG^B7%sOr3)go@!CIqRLX_89f=D9@ajdAnZG)MzT1k zCOikw9q!pyL6IZ*q>>frz!jNL?0R7E6~!Q?N_GM6FT_t|N~|_qJR0*RaVuV+==JA5 zF|BKkNbZ$wrJ+q;oAOr3u|6urNGM7UURs`3h0|~m=tqdt?5acCrh<%CP>ezP-+i?e z6#`X#)hyMOQ_M#Z-7*CYE;{m5%|Q-x{Xw3gwv>dXLAXqu3G7@}2DO&MjPF(nxuQPU zhZ#jy_3AraDmh@&HvD7q8tcSRtT^Y+;$2k|DStdVhTVL|pG*_7rZ7h~qDgp7M3FUn zuTLBD;+W`BmSq$B+npUZrk1*1bQW@6Fq=N%hR@G-I_~bA6UzI~@7I3UQJHd>=tH9F z$@2_0;zzh6SO8KGp${S9IMI)W0>MM&vG1a`CNM31rfWs z%s7kuKuI*mUF9UBKxrWCtK-5YDPx!(F*qQpaK6EXWN$bg;P`H;{_Sw%)Uw+JXTPMP zh3z$yex$m{quOIHmPTB|o)y5t7;AfokKI6hs7CnY_F%6E_p`#a0J2ZR+Nc)5!BUWF z^N>;Ko9}aoY6HgXF=QOk_#xMyfLg;_Z8m3^xgoXug}EtiB+{`250w(7X%RugB*gv$ z^l@m;LO(tG=#1Lh>M&&{Clp#7O6^t~k&*iB=2>La(wlXrL-2$1S%~)VDjeQ`4(#jU%PBkqT-d zK1G>+I8D^`Hiq+DA~m=`&;*A2sBQ1IBh)va!@q z_12HB1{bA*qViFd=o%IcSgI7mJ@o0$G^LTR@&gzgrk#%!=z4wzKGuujOS4g;L3O^h zrLF~@3l5)puJM9AAQyTJ&aiC z>-lbSnGmY5G499`a*E^v)vLxXt1AAD-L-vhXr;>Pw0V4OKP=DCzvM+PG94y5e9jH` z?$0Jc7)1kHH$AP==e{A79*d(@S^SiUy{XTN>g?<>DbqHBSXwt0GMO7(p|81twnjn< z)@8O}{o28s%wuePNEC=SW(7L43Uo!8XD)y6`NqYR$#<_lk@~hW0yL0cy7Tptf?5NF z%rQ|We_(sAe;5AX0G)CtW^)WXS+9A6U;Zb65-t)MOS`RTz1CMT`f;0061i=S*Annj zU#XZ7l$uaxcgG#|phy8rr)9p|%xCK5m5jpn8;+43JeCgS&{IRC%r)3|L{tU@>nO%z zB&ar4UWmV5M{SCym3;HP?92_e131M<_1F1bg_}u1^=7U2LMfN8jW~oJZ zgfKDEyXPO(ruF-NeP{=caqU;QcJr9^vz&#>^}BPBnb~^`#DkS73;vT*SK$9O7Xe5R zPEZha(@d^#%fW|3lbWIoMP`TuCG)FeEMKI5;Vx>08bDDEa@16klK9(g9I193rHYHS zgA?|ourdVH3Pq_DHEgNqZ!Eqe?m(k`zUP4I6O5hK{Qi2pG0^v}g80X#y!4&+T0PR} zmsB>6Bv+R_#V9qyTnGwHLlz(?qtKZklVFBiVnHkP@URBUQ(N-}>FG-V{i_5Q(hG6t zYQ3K`vQQq7gl;EvPN6J1gj*8k;n*w z@s?2_o9xa6eVc?MW8}iAJLN`6NPIOC-h%)O{-Vwk=~{g$%`#~XqSh-Z5)X>X86fsg z7ra{Nti62Jr#3;nV*f%aZq6Bp&8Y_g6-?Bn^$$^ubqGT#Mfc14FVx+O>$|39 zwx>}05zqE0wI%O4wj{YXB~CNyjk_pATZ@HAF|G}p?x+9M3ZgK}a$;P-^p?YMi0Cv} zPbEAB*CIe|5P%&u^=2hsMPG1du)!sb$Z*vC8jxz@NKU4 zOMh!nncBJ>l-6cy^Njql5Vmv}xq2J7HxwpCkgtJ}l(V z)>y31v8!0z)|$e>+n1e9c`}@IuJMap=UzLQ9%B=ic^h0$;BeA4MMolgu_PFg zNJ8g9NNmP1!hG@Xu*kwG4VnpgvSe)2WG)0w2ySImJ6@tdURFkGO#7=exWI|6>Yzh# za%eKE)MRHSf7;md-mzDx61bTN-D`I@P3JF>=jF}KO;w=K60XmF`%}U!{5JONfEgqJ z#vH#%R0dOg676GV`3wm5J^vk(<9^5c%P!*k=3aR}sC{E05aY@=s*_2^5$R4%BvZQ} zoh^S_$e!NY?msbCgwf-SM$?9b7T0f=O*avTu8RoL$cL4Mlmx&MEGH<%l1e(``7mNa z+TaHi@BG*x(Y+ZSY_HJncnAan+ZT+~Kk=+!QsQ@sb=R^DbpGwJkz_~fsD8MwH* zT*;z9Ud4nW=}VGDy1%dOj<|Z=m)so8#Do)j1yGpZ=O!z>dxMxoK`5TtQ*s8DUnUoI z1+wXJO4OM6CFMK__9T&bjlhCFcG*Qw_{WE>(tV{2;p?bprPYT^8(Jw%Y)!0zX71l^ zf?vl7t%r?ju3BRP{JeV&6a3uQ&{1zR#ZZ9|?mH0v1h-#Vh;U+XLUx;tw%p(++?Th9j_M_$4S@OIiZuhor-lwFAi{cbyVvz;`)`{wU?xJZ*tV!$*xuKme6OHc7;TcZE&7f_T7rf0iSf;HL34NZn}X#NfY-2S9EA2vYzd^Sx!j%+)!Hzs zefSHNgfcuiDUkyt0gm7&qeVhfe2zLYfRB>GxY#Z-( z(Pnf7i5Q5mshF>E~`(O7z@Kf98u+Q0q3xuIN54^`KwgiKqt)(sV8uuFU zb{~%+}F6bUy%RK92BoeJ~T^$icioj=zEM%IJZ+hZ5&o?${Ei6@w8i}>{m*a%BN!jX z9;#%|te!u;lv`XR3GO3YEY)Oc2}b=K!PH|thUbbIMy&5(VKPwEYay!gLsp0W^E*w{ zCr2$SjrWJ@jk9#`-qZ{645~pie4e0Bq-tS<=wz}{45ZT%o>vM5eXxbHfW(imR9Gxi zkyvD{NjfS2sxck%4+Q?~z39^?Pg6`SZQO|Q6?5fkzj~IgX7aK7Q(Q1Tw{E!`YLkIb zl``aJm6^G9qqd2~^+wYc4{Dsp2C~L8YedHuTZIokXG(tSZz$>sgZU!hvOgXBSWV(F zRxB*&TK)qZcIHO-Rh$%l>1~S`&F^`iU?VW_Xek5zRELh4QAdSD8X3C?oi=)~7qCL{ zQ!+Bc3{zF2*R1z50Tmp=qRO=Z3GX7`H(=md>pKPQRo(I}x=za`ztmJ+boOtbLRDZu zsZg2>5!(P&oE{PYEbZSg%kU#1?`5?Wq0+cB*Z<}(JQB7`$_O>?w%r+1qo-Xyi2on2 zwKWnX>tDyCfwFT}Z)-uwL;d?N&4%ZM*ZY%I7TeB@nL!OV0jkZ8_lBx)KTLZj4M}oc z^mng1(Xb^j9m!v`LV`cPNQ$qgp&!^3*d~#Tvk$0 zqn9IKV4GKZX&q#A*NaOSy6R0YFg&k7I{CXT|LmaSCK2I*Uh!vw|I0@6{Tn#`eeWmv zyWdN{a{DY`Iw`8iGKm%vaDb~Q1CWsTjQ(XdiuNw$uU3E~9f@^;S(J4rB}7Uc(rHlF z0Es_ERR{zpd2hT95c@n-uM?zJUr7W4r&*)H82SD6{+dxB__Jr|J>>z|=(z%|=Z*5e zEFt3zqsOzvY(M);4TP*{Edpk2@s&7S9dU}kI>xb6A?ud`7Mo6W1OknNBv@3cSUcm= z4Q+kCQDolDx;cOnyaV})PO^@jD8|GM)`)+d9&+V8w?j2UW?*fs`-vZF2l5pD_R|6X z6=}f0Iq6k%Qx9cUr8l%1PZ|BJT=x_yK@j2xizsi-kXR=%K3(5)fEOaA;m>GeJ8mW1 zc@ZYe5$Qom*M-<H;(Bj~&3N`j6kc_liPUF+_QeygcCr`a=^PR#Z~3|%s)b=hE(Jr)K@ zM&FPm&Lh&)jb?5sS1ezfO&gK|%Nx_L_yuTJYZCmsWL^=PUGZ*wpfr!ya{muvEfYL$ zc!e|mam&*)b=BbQX~0_qIq^eNNrH-^;a?=1J~o&iUmSatOUsrr_qLao@RfylWl(LD zQ}jGysVHcqk$_2zpKNH!nog0~6b8_aEfDiY&pVzy3cF~Ir<4%gCY@NLPhTmsYyuH?%X_10A{_=Bz)S1&EIq5`x|S% z1w?3i@>NThG?QK$8tQCpI%91&A%+}<&Y^5Urjn;eWhOwoXZk61<G3gS>4p5aI+EfktS#Fs+a8_5yp&c)krUD%gJF)-+95qgl6GOC=bg=_YUSQk zvz(wgDn(JJHf}?Zu$SkykB0iNuvQ56| zt-?jHyrkn2)kf~^C^#{CM!do0nemRL%?cap`aT^Yz zdN}Fdi1H4Ea7i7gm?HVE4^FO%pc!`J!-PBBuG47*x;gMpBmt#6?Oxq6j+z-1lW_-z+=$iAM!I z+)1a-Ed%|f7{*_|T&wNN%QKe5PrMO)jDle<{t^*3Sly;Q!+X$ZBBOc(R7i?LLSMOd z2Y{>=jY7z@gCrCgW-Gyx5+pv|JkSBgt#Ych&Gt(SE073Y9Y)|>)aKUge>wK|t{;=|(_lCJ zKcYamK7x+`w8v1h1yeRE^t$YJW(&p?knG|at=oAigX{c|e_-rlZTGMs%Jzl}?m29S zxIrgTfJv9&E$nEg$eb%F)1wURV>N~Wr&7Ulzsh1UTm5BZGBLe$0f9e6Mc}>~`Z`YX zOqoY;QhimBKW;|LPgv2PPY$<^Tj)0@Q=5oSjvFL8aw15}IQWjAoJSQcODjJ$IrQou zBf?H$PTG)#)L>tl%xA)^J<4(AAIWX=eJ|A_=A6W+W3SQmD?^9*cC<;w3{(NpHFXFB z{fN-s;lyF&lLV%^7~BF+6cyvagl0%(k=5c(Q%ziG3}17?ceI}=A5<`Wjqj=nwYkkpru%y=xi z(HZq?tH*3d#aoxLqHrVt2(|i;qIL8Z0hWLR&+Gj)GF9AqtC!f0E1_dK(|oKZvr)}R zj{f6h*|F9vAn|sU8$@`ZWCd_B`K%mDII3XI8`T3&kvzkNIq4RNW3)aoc@Cv8`T9R^UsF!*^#H8JVaK?HSMEUn~qqUWz37HL>nzW;vah zB%j7$c~TgoTWvqz43};_#)c9W|4{%1mwfj;Af@7J;dasE!DNTl)X<8K?bB+4_2#T2 z;ayVu`jz$BVvf#YGT^y(Mw%cL>@<~zXd5lA^!Gdbvr>U3H?chXPY+UH6vk+ubFp<2 z#gd3)#czp*&!Sn*u$2icOm>--(DBGCod1^j1E_BbOUa$0u2a%tDyXy>VE zx!V888dx$_Ms1KCWGDH~r=4Oramnojw#yJ82Agh_jN6uUIU_h>j@?(Q+UN2Yi_qyc z+XlCqL>&rAVs+m4{kX*>pRQOG36VMOm3;%mHbUD;j!Kg>VwQn1d7XNDQg|;o!c`Za z^ce+rohprRygM;i%m5(lt4Lx>%Mv)XnPIl%l_HFCh;dB*GL z3C*jL`k7`34oeg9paieCJ_K?f;5+BLFOiZE>o_;^J zkfRR9ZHuX>IQd3jZOZ+6XqgKJYgNCiE5K3>Oh#y2UANa@QY)jkUfl67DRQrOt*1Zw ze3~mVi>Z5gKJP=hn)cd|IyYYq^Z4F5#C(qrsYl6!?+tY&I@pn@UkE34f|QsjpV%pp z8}~x)!uIJK4wFIG2b|0)Tc0zDpD&;?!H8tWLa|PVwu;Jni^wD<41qadQRPF5L>mS> zP4Q*g-!^+U2AaUL$j|^P5{HxQN`jhCKynqbI-HiaNi9CmPn1GzZ>x+goWh`!`BnX$ z(h3m5mW311;545^3k$|@ zj4VE))8fSE(NyqSefJ4QCr4j-gN!|j`sVF$;qI`Mt`92zzw-4Id#vTuc#%spA3fy)d%?XMK@P)}2 z*c|ZDHN6(+lUm?q*&AsSuK4gydK#suFwKue3Mh%s-LkOjv7EHjsfSCnAv1u?08U|w zZY3WdKHtmTg+!G^8!Z6IRYjudi;mEkk)9wnLQ``yzHl1a+`EsVjc8HT&N|9F!B^@ zX9+r?ZU*%%lYkF^(3O0Pghv123;YKgL_qb?hu;;{T&9LGOH=z(RK5!=D@Jej}GzH_D)OI zI*`vc;m9ISYxO&uTt7g&|NloS|GjzX8tk6~n#SBE|}9m7wkzxgxS zOp@2Qbk}7idsG{q*zBA$LEe<^`7)IC@8_S1Y2st`PnfsgtA|HLY%kN&j3;+vo!+r8 z(PZD=88p9c#_jA;Maei8^R*BUj*fE3%dXhGJYrwt&I(d`$T6zb=X~mx%64pdmha|g zhm~_8mOyl*A;~j#!$#BGp~MB!aaYvISsDrmp3mw|sH}+$Z%4DBs~LK-4#KTNh&rYR zunmO!{XYQmKn=fG2`3*ugn>@mt!8Q(d_L+PkdLXL8@-Ck+=vnl%}AJGLr0XJ9m40y zuYPx-!wP!wp?wB)bkTjE)O`Q0KL43@@yTaQdi^abj_4bO9B5AHHn|PPJqTdr30PJ& z`c(J;D#VbXF!B3{ecv{+1p5X7J2EzzGtnU4o{WoimupW*&^^E3&(^J5r}XdNpB=NG(G$UGeyXV|5d!srtNBsx zv=)IWIN?voW5Xwcy{SoUWxJx!>w>FYhkQNq66AZ4=|vfRAAF4av~r~_qB*I&S50O1 z%7p+(6f(iaTmurHg>5cq1+s#VhPUQcNx4JD(kdv%5SF`K_)DhCqtH8~ahjd#pl)q# zM-{{)M>dd&j3iTROHhtB)rZ9&(WlfW9P7=Fop_fWIhbV)$hR`OXIW|a=}MYQx14SV zViAJEB<9mVY5brLLzWuaRl%#am#XEfcdF&fH?Z=ZRVX7RdM8G`7f}*BMaW4_kYVs- zBuG*U?v{cxNpH`O??^kc5+o84uT`u7p9t|KHUI!X07*naR9AqE6vS)zI!U`B{RUt= z9_Y4I^R>P#kp6Py32TmMjI>!8YdU3MH|FV|W2N-WW~p5=trRS}rnq#)p{thegaZJF z&U6H=)IpP52p(VtO~^?I9^|=d1PyLh2U)PFLWRt*str5U%GE`zYIh~1mQsD05nqP? zM9LD8ACi81hC$DVt#ym33HGVmwOs+!Cnci^?j$zsZ$Y%h;Q^w@em)S=H=z% zVd3a^nOlL%GL#>k@C&JF;8`L;CVVin4eeMsA^PM5+E=W~%>`xO-gI6RNl#bvLrTDxif+XLwYWr1Ar%$RR}IAk(d15XY9`1hhgVfC ziW04BAmbwPQTfBa1|ir_WPF=ws3=D1s?in89{XR~h--c#vwhnv+tOin2NMrY@{wjR z%}7Z32!UB+*;>efXTj7{@p4T?yL$nH3*;LNWmI8?O^trvygwb%qv}^-hM8zWK=nC% z+qV;^u6zI6_xn%XCbXvU3}Cqe&O$-ZQUI?YbGy)&y3wb?bpx$l+ryg#1Ot?_e2FJu zySBJWGy$NfqGZ}cW^Od}C27Y<`5RDl`8iPb?~TvDlS-&Lvnx@i~NGH0c>`sMlRrm7%HLn)eQqBf?swnS@F z9X*28T&8ZL#Y=H?wERTM9w8wv2myLF=-m}UCW1dKYOMpO`93Kks&%iKm#>CGUfrESD@H^za2Hn$ zGBJd9lbwjlFJE-;^tvY|+$XF0msie68GhLhdPB4Ws(Ra4gweeHWL}UZ!Q-kgnv|mtIJC@YX@=lT!CAPRew4 zr%Jf&b)XM%wWR&i4>kWAGHF0Y#`0~PIky_a$bAOl%vX^wMgAS~&&ctfNV~hdyc|*P z)79|@SoyDhr*7TrIz{`|CE}fW1Tqp1SFu1K zz(T@5-2KX zYk%CLe*gLcZAl%uiI7S}1laEh()}bYP70PoTAofk#FQa;(2N`fYGtv|icIJ^Xl-kR z$;ZaD9!(I>7Boorr1?rDR6SXs>mOUda!xwP7&NShHK=O_>z$Lrd%B#QWU7f|L+*P6k=?$9|E~G;x)od$dXw$}B>gE+&mCbK0($_BE0z#frM22Xeo|VQ( zNqd3ACb$hj+BY=+uhovkYomEmiEh>TNsx~%3*bcrBt1h!%n~Z$aaQM-ImYiuxDcU3 zI8hKUTDc@A7A;MY=CN zr*&;nu^~=S@Y29UD!LNl5D)IQ;N)2$AAsLE{f*E%5sYqD}EAmgcf7Lnkxr$WZyg|J`(g z%M4IZ%|uRW_o{7!ZeDQrh`Yz!yo|<+_)m`rdIJv6c$OQ>*KX4j)Y-P3m_(qDl_uGV zEDy*Qfi3Y^m@f4-yiXpl_%(52epYgJJgRvIBqFUnJq*8j{5TJ{V)8Um%~4S->k5 z0!|vEq!S4@LE;@`6Aksq29m>Deq#)uI?@`~H;*5rtK1!d(t>Hz4?iqdzF)SCeZOoe zTea$EwqgASwsq?khTkHGwRoGIoz40W7{~?=9E1QygV>0X2eE_Uo&&C$TwOBQkS-bK zkYftWvmZG}*|Bt^GVh7c^f@2@sO}OspmImbP7Y4!pRFNDhUB)%xs#Q!UeJuB06jAk z-8?%?KGrq4ul4t70uyk0Ag4)-n24r^+K%;@kCR4N`DY$t9yp{2AJQ$$8U(@^nvHc6 zX|t|hzg}4dpT!&2uVWQlH!1$?Y@);cQe^>!x+1ElF3V}*p`jE)t{GbP=9`7q4x9KUmzcp zqGc&UN=6gh3f`-Yng2($Bgcd2jyYeXkcZX;@S;&E5o;Ry2&ykVU9=)sPbLIj$@+)z zpngUJJ{&laBM?)JQCt3+rs(Vj&N&I1+FP4mAgK>~T~lm>yFe(oRHp@ptqLdyO#$H` z92)r6y?r8zMZIyNugvk_wt2@Oz{+P4ZAd%d2DF?&#%f%FR}fB7)+~&YVHS`O<5e_k z<4M$xn^;M}E1OJCEs&A$^96J48Xth91sT}z@g!U!DdfYydG_(9uXHYHLsEa94Dvy8 zfUZDQVRQ-2Solgc9T>O4H%@IZn!eD2JZ$^?qwPb{)hBSGZV?)vlZk|+YEoZm!G|n7 zz`zxzhB<@jN(Pf`=z+KVC6&PPkhE2c*5092cN+ADpZv@eODsyQS+gc7;2AyU5MhQ5 zld9mguQ}hYsWeTSDT1X&1A1t%D9~Eq7^oRQOUERdL5oL$7Kg2)A<+9^nYN5b$Rmy% zN2YyP+eqxkj3#%KGCVw2?AViV8f)67{uI}g2=7ivKsv6$CiR_XkxxaAr2!c=9trn7 z$VVa1KrTX#*R*lN20=ph%j5ZV>d?-9Oh3ONP?+J3@;zDSW{Lj++FwTI<*xHYP3?Zc zHoqpSk?1DU7Np5D^`?<`P!uc!^O@&?GegH7I?y^M#Z}u@TRA zk8dncKfLuFeR<7ER9{=0%_i6201Us`T3tlLhLEQvvUxUCe+tm99*~S4ydxtf_tE}7 z?e``j7145QpzW9r>h}5*jDjQgw?^*Ql^>Yu7H&UkU^Y#YFJJz>vS9vvwhTn{&+z$* zfN4r~HP*d4A<_H2tgKArjT<+zMT-_TxGTv|pFVxqkbAviQUA^+o^uLR}N2r-TxrWIsbT>C~9{KWRrQ7^isQ0zi#P zU!<)I;6(uwLy!$3AZ-^H5jkmqangQa%LHjbdO#)S0;@E!XSQlAWa3Hn;fxXlSSv+P zR(g1d-mH`AlaP$e1?!k%E8NM+i7%YInI_swhZf}GhlcY&Nsl^LO56|V^bJq>|86-t zd*ZX7M1s@^qR9mD!l3&ICy~IGb4SOn$j1!CwE7745q%)VNrzQ7h$rxok@8h8xMK_g zx8y`VC-@fnEWV8Lg@uP@jr;4@NhKc!k1CRG%ZtKmh8nqV|{&A)Ae&P_5MN%#ce zS9ATldtbZwk%C7?UAw|0w+wLIfB~bflGzNjMWso0!i#~dAR*9*_6|kHPH`zYG`L>7 zHaSShGtW9($iwRGZ2pq<7{F7q2mdtGrzoc-c#{_6+j4vgV3b8(zcOgFh2Jr*z_34T zmi^-Y)QvFR3qm`>Nxl*(NKGOq(k!6_KWw$*2*SdifD<|te4u8`k;qu=P2KwAJRln295~7IF zkwo~?;*CAVvgfn*(pRnaTlm<5ZY<@&bAl(1>BoyYm8IpwBRQak&geHH&2;8>kBr2{yK8`PqPl56T~X z_(4rk;kJbxcGy@pcI@Ho(6M9Lh!G=snn&$^$N-KUocfokO4*v7>XJ`?P`nXEgJew7wD z=tqe^dY|C^)(+khbGkbx;vv?6RmX+=8OZx3yyllSm1D8$V7q z`LF_;zc>c<|C%sJD)}He5IsR%IdB+$W$gBOM*|L8ajTIH!Y5G^Dr%j!?yaMVgv4u& z^qr(hcA_@6tXHcLIC79h1fA21~0Nyu&6wh71(c{ZIUk2Mpe z2#_?qm8n?-M&Q>^k%{h__U+q6kuy^`kt4JiL!+OL@M9{p zm`Jw!k_) zLOlJvawU80uaB~~-kiz&@Y@#)iL~tAd}F4_y?ggjF8l4}?4nC9<+-Vz0aqSt+;GZ4 z)(BOaq<*)-0>%niFd}39_G|9~Z4@xE2;G zo)5!nW_70T$A1k@9dYH)RFMejMso@>QEycP#WNC-vWM#&!6EXsZ~aGIxqo+`$mb;4 zs^tH&`j^*wd-pqY6V*9_OTx=X6LN6P_kSPaPJCaGb;KJTqT07&W5MuU%hQ%+4=ay0 z$%c`>Q!`8_nPgX>k7d-?&cSP4CW$~2ATYF-CCq9gX10l5WlSPjZ}@E@)4_ku)_)T9 zrI!Ismu(-AmDnp61msMt5ZjA2aO#BN9<3fbv2v^l9iN&pT1lm{Gbfez@6)$$qQl3E zigt$U$hDz;SZa+XX~NgMeJXe`F7qWajs9u=5D$sUNvcV+kcqyZgB(w3$BrGMxM?oM zCLxI=vbAk&VyK<{n38Y8w)iKECrqQunwpX#J~kjB>083Ok@UL>)_@zQgf4^mdQbZK zrxh#MJ$K*9K6wAVn09HgY12k_^Pg{Gk38~_a@Ez>u-{#IHSdw_9QL1!LJwR#PI>o? zC!BAKsM?Ww+tix}Zwz{0)S71x!Y5kfipzjCVPAjgr~$btA=B#+Lh#T&xhtWXHDPj3 z3s-+R**NgjgRP0ZbNPW-02LHU3C!Z(efJ#@y|1%xzWzEXo}Y?}3ikF}Z;GtzIvacV z;cV(jzhTqPIFo|l3)PQreOq4!Z0$tyMcLLawKWIHdm{NFB@=ir z#xVQZL_69AkgelLQn6>stY%~tsVuq?S|XOI7J*3n*M8HoJ;Ii_0ekq-tmR5{GbpSg zP06;rnxSeAP30~J$VkL^)fKKNr0qne*wrhZ9D@rdV?*)EpPoJ~ShQ$~fL8u#6oF9j zPeU60A@RTD6^kD@%fIpC9}>T+d#>SuJC_`cJ_w_T+R$RwLw z76_l2mgO`w_}EpQj%-WQiYr3Yhf=n?>ODg$kBAcluDkw^?40T6#zkrKyV${pJG!tv zvJ~OyeG44>Cqo)Ba-^sxB3EwSTE-BU3dtrR$*YbZlc`GNYIw>aEtT+M1iFH)_&glH zlaELf=c%A0ZBsj?#0RZOq@F#*e42I>HW`M2=(D>KVK#>i>1JgtS+7I2??{ul0+0m%(EloZa3aKJ@6)J!XI1gq5#rx8mOcY zXfC%fyCx0BG<-v5!EV%)xC=6+wMnvYMLeL#9zNF`d(wf%DQO-q0^tD)K6~~o_RLfN zVxN8b38M(-$(rcBO4`$u{{oVG%G8tUn*Sef4Lk1$bK`rLIC=`UGn zAS5rwCFGftfeelX6YWM^09`W9>N|a`L9dY(rxZ80JDC=rkyrJ;i!4n-CnMnctW&y@j#hv zh!}`jG&n>A3asI18h8X4IWI!BB*bf;kWg*)0AEdJW7FSDEe zcmo2%6xTa4em){9Gdtt-Q`sq}p2i+~{O>$BBV*ummxb*58J#2m*@%`u(U0E*Pm6^zPv~%j(sAfcog-6k3a{uVrp(7@b zw0;u_GDH48Uwr;K_Uyl(RubKZa$8z65D3&U|G(t2%h*L1U&^ymQu>^KxN-IAqs(pZ zF4f<;@;U8mLE97FG_51z7p7g}ySG3pUAm4iWbw zf!p5HzdC5`hG^6!A)C2i6bV_Vu-;C^(hZecAx)=XH1Gw$RYlbxZlnomlx=o%-_Aa` z-kw!mo4Kj{Ng^N-NL&O+RoXV3N!&U%RWrP$)vi_B`{!2eaPIWv2jDLjqdrTes6B?M zYMh5O%K>8%^tEgS`dn5>#Dq)p7Wn|Kmjd!$YhD%iYOyt9nZB=6I#Bd_ggOXE= zA|a#8+l)3y^kH(paRefokp2K~l7zhN6l3U>#~9Z*b(T_EQo?_G*(J)yAAQ*P1>^8r zGOJy&NW5fW*a4uXgR`4AClf`}x| z`xqT_Mwwrmc!o5TT@hZAX-&n8MY%?!ofrPb7=H11b4Hq*_aF>Nd-UPeO; zejce_9hu1d|CXEo%pQ5@LFLlRE@zit@jI54o!#x6!$OyzFx)u)AF~`UKKPL;T+u z-s9&e5)#Lz7O7sIk6_2(AT18#YaJSph=9v&ElA&*mr1=N-p4n41VtVo5s(P99f1jL zKQ$rFgsp-$ef6sYA02-63TB}1#EK4VfS^l`m{v#u9sR0L53k`SM3rSF8bM&UPucy* zr|POkeu|H;NZ&v#f$x2Bp`9SK*&^CPdw<2)zCxF|li8AT)vkGLxQwgym2*f&~iBe3_Si;y~5DLh%%v|0t?EjT)n z-n}IDjXxG4=;~Otek3U$j5czGyWDdE3W|+8)J8oH?p!)xZwe{Tl?X@#5+8xF zsA4;D7jX;L=p76Di^7vi2Xvq~7Id)DEUs!^9r}@Z+aw7Yx2@a8f)f#CU?@#wR4nF5 z+3d&H#6xT$qFuXaO?aG&AR!|Y^)@0HH>YK13m2HI#HUn>LQ_9LT8%_2l42LpkabM4 z8>t$3kBmwcK3gHonV#~xaKPMrk91kfceNE6BYb_#1PIL}3E5FAS)> zEE{FdM@L3WbeUL)^WZau8L9F8NI_@byec&1#mhqv5&?;4jz0m-$B_pPWwez9WQ`QG zd8ewX3en>qVFwNyqCE5TQxwFB_sg>mdExTV1Fv2YnnZ0NE>2r&8ICq`2-9pD=aK); zTr8$$F=|sHmqmDhHWx>pn7{}`NJvIOX-ub6H4r!fBW019YQdM8rIH_CiuP+&Z|e#y zXq8?jiGV~vBG4!T6q$(T+{v0S5wv<#iM|q(5}<&E9IjZf$X3My4=wc2f{%!hO+Hp+ zN_+%};3INR1^HwNZ8$&ZsMyM6CTwDYr-C9=#6&M>YwNY4c__wnRPs1%O-Nd4Pln|6 z+RA$%4*fOtCF+^98%YjqZO2w|MghYnAze;#!<6`mNF7QkPK8_q%G3k`G>Cr^nS`m% zL)XOd@%A;nLjQpr4^kf?&hyeODYNY?Xc2R4UsxkNa`_LgHBKEr#5{vOKXCuO?9z)b zV!>cM81P4G(Dv;=o_L&HedQJ4Aq5e6;)mA;Ppi2aIRGJY$O5%z`&7)tr&KmRmPnx} z0+L88+Y1A2Qw6C-1kv5Q#jA+ifqFe{s@e1W2|0sw*yM!}l91&HwUC zCQGK9ByU&W8@lcIVMd483#fc-d!!h$@x34-W8jFBx)7*P$A+T>!(S%5b6TrRGh8NY zn1Oec4Fl@f(GEZl?jRDfg1D^f$>KMGfIz@;2+%fb7sutgYKE~8l=1@w?fJ%yfh|OA zd5t$?(D?$^> zJgm!n16Uy=-3|!;GE0aw&1LYqsFnpKl$t7H2^6_Su70XeF#ega)dHtimLG}aD z4cE?wPLUV39?fg7zEXPim6uyE{myyyufP1lj+uA_JLAl=*xmQuXXa;T_I>{B$UP_Q zE5CNildjJ=683!0K-tNa+yamQb9a74i>qeyREU2Gg+P>qoGvklFB!}&n^FW_mP=6# zN#W8vm;sPe8VY8pq^)%({tE;I0s?`Z5$K7p#4ZJsa>6C;I65>iV@H}i(}0gugoo^% z>rBdjW08N11Cu6n+@4uj5D-# z5Rev$6(b-?_4j|*gijgP!#Ii(v1s8p?85WUmDa6W7w0sI$L$J%7yt7D`{MJ@rAHtC zHyeM*q3$F0*3Vo%-Q45&$KB8Ty&c=uf{5G|gyQ#V2vp2l%HXXP!Nl%^YHC2lRN)~T8$H}Z zHtDT1YygRn+6YiBARCY*0jXmFP>ywU1lAT=0>%QQO=ZQ9fGlUrNm61b$0@ax6&4lw zn1ULR03r{GCXQsuDLN^|LV{~klBAFm0@>`Mkfy?}cASFk?GZZ~ae886KPDK!IvK&O zN`i!}rKn&84@T5dn}DRO(;!JYewWjS1P=akYm+70VAoGFhl*SB6fU*MLA&64Ls`^vb9>*uuA`&2)?^ z6Pe|k#m-(00E({!0s?^w1gvRd#b+mPS_FHyujc?>gkX6CPj%pfGglzRth;IBi z86(H!ounbj7N8FLQ3e_LA(t{tL`leKk<^!btW<&p$XPyL_z-F2icUp&U6*7_nk^XM zLM)YTO3l-7#Ii}2=87%^Q!|^pHCMrqMMA2IVd=K-*0&|!!-?g(7BSVOAnlT&&XP#T zS%{XkQ2Up}IhMnr1BHw zKNnwg0b9Ie8Ef0NU6(hfg(r`n>3*GOhDagH!!+3{Z9!>B5RgMLCXyHI^eQ$WGdO8w z6Ie*(zCBs?JJUip=46;%gTbIl(vR=H^LA2}O)4h(sVARcD^{$KUV7zK)}cek&QG5m zxns|c%7b@)pswO1BxSkQr8081PPf01uBv#5NJsR1PNJzew*@w ziayic*rA`X(Ze@%NB@%Tmfpuw>&2q;8u#1cyjo;fN)_W4ai{~Z! z&(xU^4RuNaCo~kTvuUcRv+*vbtZZv`dCX8_id$tFOP74dF28hIykC>l&m%`7mt0I% zVi_CO!<=x;5QDmis?v}qNJtN2J}1hXyZewCdFb{;J$73TOm-q5hxcGP?_L_Zg$T&a zoBv^ljyp&Ykh~xh>)(>^7PA9K4rkwd{WT+7omWrLuX|{Uwm(m$O(?gJM5L`bl30j| z)Ko{H-W?D&Q4&~cqk6W)XMun~pd}CxokRud#5GoY5@!Md2OywV5RZ0lE&koKNXT%= z{yL?PB_$<1f7u?Bv`DQ$lX7y{LnJLZ@UgT^qd$s6bp#@j2>%K`ekyO_RKLuzpHD~5 z$zqa|0_AsV0%zDJ&fTgZi1Rp?Q+76axPXYgVrFjJwv|mib&6xWU^fKHPd}N%C_?o{ zz2i7-O6*=5f5_W3nM)ztkSeIa0o1obIc%qA#l}|P!mknksvuyK zkXRuL60!<13_?td9ggTUU?{%_BKAZ~#1M{3Lb?@8@;edJn2c#6M_-wl#9d+M2297f zXP?PPLQaI55gA4Aswm^}@)ZJb`xk)+7b0s;YnKm`JVgsi|~Q%0w#Q8V4-t3~*w(O?x4_VKJfHZxq&*H+vn&IAG} zg@Bu^Lq-D@M)qk+AIW$-g*G`R`3YhbLBl zYP!ye-%UWKX+`hUFflnLmzwOVQInq`Eh-Isy*!Y-+p6n zzcripn-`9cP3ZrUf!Blo=Mx=p-=H!2+5N8SP-qNshXyaU@nE(*gm3z%CH5IW&lARem(xj>Q93nkir)qjJ!F6&fmEZNj8G=pNK)% zttFCD;odV5lbLjBvYpcWvU)X}ar4f9ih7Idm0ygo^CL(a>jN92$ zPyCxCBBfJDY8T#lytWU2WMv{!M_nLwd{WCN^?Xv(XHeG{!Ep#g3mpq#EQo1gObcXM zDDwp~7S5Hmr=+x;tw2aDnjKe7M1}OPG?*tK`qI=Zdy%7R2n6*CzC?ytn-jwCzyHp&*(%fQ6w5F51hz`H zj;Xh>3mJo^rW3=oXf`1RxidQB@s(3?A=!w75ED@q-QHwDFXzARy?h@P_1hG z>%^S`0f9g&BS5}y*QIg>>ybBfkmfc|+bxjf;e90ppwN%XIt;=^D5~gXl`teBvwWg2 ztp}dslH^9fAFSAyP~m0731Ri>RXm%S9+uy|4fjH7@6M2aY*JA72)mH=s|jFp1ff$2 zst!0OX4}c25R$UkA2b|$6oj@ze8bN=@+or;je2#_h2 zX%}6{zWDqzCQGLKt`j0x^lOi@r$9s`ZN^%*Nl7iABqDR7>iQL&6u(&fOLYV)3LFb! zS{Ty;*LZ9q0xE@qZObl&vQ+=9C_C3@t1pnaSepyR z(wq;r`!$z$QYRpv`gu%5@~tIMPedGS zNl3{Q>!f(ri2``To_WvR@qDBsp7s8F@9=zf%rpCu(-;218j!8cFdZ9YyVFTpWhqFR zOF??DI8Zndg~WJOCOtYeGEJ3D@1lj@um>NQ<%n`|LMAm$W8^@J?0RN;%y$31G<;D^ zdc8s-*Mk|OWha!>^+|1CNJBcIdc^W*+qO1oL0mBTP`#b2t%e12gci(_r0Pr}n~>I? ztGnI)e}p%E6}#DqCu*yK3*8+@{piHJ?<&vXf)%~At>0N5v37NFivF&)9Xik5cE2?# zn!S2uYwM;3gO#Gy6+5r>x+}FVTGW2TA3R4i;e+CdWN08^OdY#|x=(|Ic|iw3KBiWY z$8tlSJRKPdFrCcD9=C#JhKwU=QA7hyG@#;EReQ=||B|}LZx?;Tx^|tyM)WrNA(ub% zg_p+vr=d^C1VWjR0Vv1BTVwMPMGy$o1A#;&-|CU@TmnA}Gr@%DoXiO_w#7f=MwJ&L)gQG6h7WsYu>F zC;w5l;(jcv%^0EG>?v;H7iJDPZ=Sc$@f#~A!&bYj{GAidWx48VJF@-fKevByxT8tP ztRdHZ?&&zPihq_=uOHRx!7sn|C}NAQbu0-921a|e_sMH{AHW@ZfqqdG7=$#@qeftJ z7lfo&j^kv{3B8TTW5Odq1Y`*FhIB?&86uz{(-5Ip!6Z#*5zHyv-gHYCBjNKBhXj5( z?@M;l$y3|uUDN8-g!^*G#gpo5cfM`jJ#^KfC5h;v7uN2N(eX~Yl z)rpJ)Mtq0vMAf$h#dXbvz(PnEf`nxI_B8t9ofZjcX*7~rf*Yf>$_g+_$2-&}1qta# zdewe?SoXpFjbrKjx@)eIgd-Y4BJ~2<_k`DM>CwlZV22DeCQlk{EPHRIyp`sKm>MH`)5IDQj3HGncN*vSz4I$CfLc z(~yU3+1r-3q-YM{ydb;%X+0;!2uLEAIKc#)nec{ZDs*&0jM@o}kSy$H#(*ZbH(6 zvFt3eLGFz3$()bLnSz;&O5|i!G1HeXTgqO0^%eVTaZEu3Ui{AsY~H-N z2<#>Xn5E=%&n7-b+`gL;s6;~IO>j;Fc}G2BM6hOna~cxc;a*+gSRuNYaJS!YIj*dF z-)>H1f;`@Iwd%iI=0@Hu8<|8rg0MHxu;9eUgm@G1f>F(sj+95_RZ?iDLe~j{7}PKv#NszK@mSni{2T|U+{Zzt)*AHMsSfyP3ul2_Z_gDN z0US37pyJDm+wH%SaG*XDrY)d~4FVG8Ere%mj*9q~stC+SjW34;B!eWR0Sb~7zM@mf zP*@63wzG4bLqd`&(kFBN$H&5p4%QBd6+2D5$vKV5cm8v6MoQTkAR`< z&`FtmoTE**O12=+8*OwS(8(Bs_F`Up&DCT(IZm$?k4aPn{&ndz76=5)_BlrXXU>is z6iX+~he#fhHe<$ilY|6#}O6?OWcsrQnt+h~g>1`ru5!rBuOqUc!??mzpZUwX> zF)5jQO!Zn-204w0lsr8r@hv5rU;&R| zR-Y>=+7~795n`WEvxK}fC{{3#F5S>oMWS3o{}O8Y@AaIxc(1~rM<#I=8d-Ol zmZh*;*Tc+oMc`vy*WuWefpH_1TRE%S&xY^sfxyKnM5F;#+rrF&=l)6L0+tM59CbsM z)d(y(1wlpz%rFbV7_~;6g9M5$73&Og1_=%o!d<#5MNBvH=s`w3AluwfLzf^D$2?sQ z+Ae$B{b>ZY_Hb1F@U-6O?KyEXC!bJ1VCn*fRE;ng+aQ1mnl9835R%~Z&1A11ydI$K z8)2qG&@W9g6fZggkFINwGVp7s>aHL1f{3$t{N6j{z>kc1=~k)+7(!lI2_ z`_vFRcI?PL_-IZIzogbz?c29ctqjE}HzaE8^gx}9;OCuQ3q=0wgFrYOW`F$tJKJN= zJ=vK4TF3co)Cv-ksGs=LJQC}P@hgPUjAS8U3mWO9@r{pX#}6|)^lVFZk|lH2teG{A zarMcUxF-1!c;=}m*?H$(z8c)o~YRP?1T2^$vZTUmZvNQ+HR$vz_0%2ebhJ_wWsq#}A! z`}Q5;^S~r}%8fT%&%T&9-`t}E8~xA}ZN(L@sY_@c#j{|5Zj_*kO|qjZmTU$LvwJc5 zmhpWx=@C229*>8O89UbgD2@(8fGqbrJ<#AOaBzAps5$>G7g@D{S)hoZ>urSo_`@P0 z`?fR5Hu2B=dm4k_MG@x;E8#~PlbSv+9gAlX@u+}+)Db0&Y_y>kWgD)ZpihJpr2O$m zAF|)ptl>#<;=i=CG^S}9mr`{~{wcqA-8q9j|H6yxfWF3&{o1q7e%S&o8LErcMy#9H z86vQW$X>4qaqlRAuAB?B9-iY zF8#Iam;d|Q3>0+59n^`XasZj&_Gj%s<7=KtE(Hk_77|yn4Z;p2&4r3J)$q9@W+)Jf zK<*u4P^lO~km~~L1d`1q>1u`w7?|QxA@JoQ`bj38V`M#Z=uImEzx{Ws87a*N45?l} zASIhol)2xnOSlvy*~o-5CtwnR{m^pBh?Hu&S!^=RFEJ$&4#8DPveysP122?2A}%ux zT>>gRgnyaj0uku~>qd&0T-orJ;nTl5WWyInxQz(e`bg;=Pc(RAtGC_SXZE__cU}*; zDmH{dp^A^+LL@0|ghOGMOV_B6O~vkk@+3UcrE1AGEnh``shFP7zJv2fNK!X@_@M`+ zE3UknojFRsXzez+=&>*4-}%IwK(Ur*Ud*yc62OCR0(|Oios()dHu@M_utwhU($}uX z=oTl1O~od!J1v!G3|t8{cF}}0)hAW;DQiJO)(553zQ*l#v!joh#15Zu1lxP>eOUJ% zJs5pQfg}Le{rxvvxpD>B=4Bte|DI!s$+c_$tGy6uXyOe2b%PzWV(6CkfH48Nt4+A z`w!=y%GA9W}kifDSPRq|F9j~wJHf5_B* zhUWLFW#N#|*VqrMowY2MLQ5KF-im`=zq_dD%l zJ8QNoHpGU*5e+0H`ic-#v3&*GG;a1H9y`wV%M$2_ZHbia`w|G|q)W=SQj<$Uc5omG zNriRC?bBKR{sY)yhfhembyE0RS$5y|$Y(Jkl2nn%RgH`ZIQI6iaDzq#{)m*MO&z{k z`F$jCl@`|^0=Nhh-n8`iVSF1eV^n;TPF+WDpw{f5X`3T?7s zZI1#;o)I}{V5}h(*bt7~4j*~7h7qa8hniKf>1_yCA08PAUowXOW@y@S?e^X!Fl;o^Rt%b#HLFOFzKy1{CkqhuJ0VmNxmn;RKk{sOX3x`U4K1{t; zK%Bv{wu`$H++lDF8r)@Yw-5;K?(VKLxLa_7Tks&k9fA|w-JP7t-g~X{U(M}&UES5y zRrNkqcTd6NOkc6k1Q^DAE9pail2Ohp_4>dSG{c1Cef=?_CSz3OcjEZl#UoqYd2;B) z<~_%*hI1$@XsC-KIp!&eU z1o7-ClQWk&;|ti{zPqbD4W$DilrEoYILZM|*LA_Q!e(D!I@dQz_B{_qe#>gxj%In*CHY)^9g7>VB;g8<4KU{;sj!P&n-pvQUDjK=DVxR=h%l1%YLLGi+ZHVIK@oLL9TquSQ&VFk zZEg3rR`7+)dblC#@el1`a*>_p?Y4tI5j=cK2pY~tDO?CSHxb(tzfE>wtVy|OVER3` zkcJ6#R8b-d+9wx$cv@nRhL0GPC5m!jzaF2Dw_vX$MHDnpgs~^(9wW+0T3~eDYi;z| zI!0-yYYG7~eZRiPu)@pPO6bajDIn0AFbP2VbP5W?B^=1?>Fcm3_N6LGCK{Yp&Wxt= zCzpaoK9?urL_{|j4TQEp$qK2CECP@#pAIpBJc0T$9hZUkLOb!+qN3h24k{XBok4MkAlSQT-#t zUFiBlNd^X)|DHyt|fD-7ZlAlf}?&9}PpbzJ1zvIh`$q|UWgw)tP%jnVA7fLE7$ zTJX0a^m&!;r-jvV#~uR)|F_>d6Iti`K1BAT-IuR7XisL581optQkGPKtvr7V?U&qC z6N%Q^X3R3mGFeIIxpkJgAAD`7sjAA&%8apQvL*SBFF5g zzVz)EJSlzrVw3m+MiRqPAX{!0N(LWOO2>W2(F^j4$WwjUMQzt5$8S1{*>nSpcdLNK zP*gi>UF3Bd4;$)DM6v0ry;5s9i2!^yt=YnuH#XLs+-VinnOI7!O{8%HRsu0%u}}_= zX4eBYySHce1SvYmP-6|{tnu?q{@C2W-tg-~;pP$1HS=4M8}Em?v4uVw0DM*yL@fh@ z^QXoMp0z`gFX?);D>eC}3B?;BWA~6|y3QJ8+TvMur+ca`Z26lvlv-c6;|L)_>f=N}CLXThSWtO#sh4UC=k*n&QXQg>UdE;i=I=zm>@kh;d zDx-h5_n1Xq5BI0BNhhvlikg;SNgq2UO!rEYe2*Xxs4f_FCv`RK*2&r1PjrX43BvDi z3yqd4Kue%>#3>&S&7Kok0ApJKYv_XuJTxs_j@AGhR~semJ9i|omlp6 zv20|EfRJFNS&pU8>VXvI%L+rLW+!tN}ns3{tv<*!$PH3=sdLsO zH#QbM^aM%QoK=up94tbr#|&^S~n+ZuSEn#)T2v2%5`R|IZHs6WoG~Avg88 z^!Ngs;b6*@gt3(V@$~R3_>2pMNi9&mfb(du(2hA$AEQjW-=dXpcQ(39^y-IzZRgKq%G0!f4vQ=%&h#ep`aX`t=) z!&t4CQmPQ^zsKSRL&*l6-f5vi$7gm^t|pzC=f2z??dakp8Bt+^7r(f}zihpvqE+fx zq%WNJ{(aGL+UG)@FLMmgf9oc8(-QqGRg7@V0hjWY;hbt3L*AE!qk*HiZu_(Q7&hrS zIl)gBGzPUQ)lyW+TAVCkd#56FR8}oT8j^tnYyH+ZSzA15&RQg0#eKOy|3gS?K@Xh` zy#aQ)+aD1>s*Vf*P2>x87yB_ZbjnSJq^pUzq?%}K0DgG*W5Ck|&R)O5psg>t>uG%g zH5IDo>E;UJv;^q>e7u36+WgA%o(Erdu`s4f);GDG<^EjIagM5_UrknN_euXbo%557 z*uuAjS*7%z{Vd^&&!*$k{*i6ZcNbFudbwGvtg9^!=8!^Bz1#f2w`Iou<}G%1H^kAH@*6NWUYo`Z%0M1{-zt7NG z_c>}0q6R__o!Q1;(@qlsFP%gVFlNG*yTU3}sysF7qSH`fV(^M&B=J^U4zMi}3JTmd z3d5c#3z+jyKM93d8|{~nQW2*KQaI%9va-_Xl$3P4%Ipk1Zrix!rM;2t^#la9x z$`w&hrp=!$=Cxx*<0pg{u{Ld4JnbZ~qn*fo373Sf9KL5}?-|hkuFEF;H;}#I9Ts8t zi%*ot85QgV{DJU=&)!h{8ov6kU-v|E_f}%*YfGSJpoA_Ahfc)FoY;8krp~p(v0U%f zV2%Dok%KeLthYMIgC`?4x$qt_lFXV~`iY>|8|h~~!4&s$P+WzPeiMxv!P2n=1%t6* z1k7?8%@DJcg%rG5-s2Yo_Ytr4Anxw98Ux=Jcji`{K6aJ8K(lv9tpP^HF!RNOTTEiq znU1P3CKt1Kq!+znR{JK#f51>iWbj>ix9eui+cnwe{A<2X!{5uT*M_OUw7?hx6<4ZR z6Ur1tDU@C4&)eUcfBiD@`+7xiW6zKhaJ;fY>M~3fS;B%Ww;cGl%aaHU3T@AL+4{Ce zn=Ruxh7}(>WkGz@_V&_K5_o^g5IB&pJq77~;qpzyusDv%KZl8&4QjHUz>mJ?BO!Z* zuIQxtv?wF7O;p7JCM%E@EB@IdK?H(-QAEdMGXjw5m&Ih<$4^xOx1ulbj#H-Z74-p!rC&MG^D9ow zK@^yrn0Ddn`tlP-75gfDJW0s}NDSklom`2Kp<6&nrX-e;(n8ZhJ8bY2rqDy8_-de^ z>Gg_dDO5snH6oBNtjPRr(%;d~#eB7w_V;nHMV7!k;J1sNa5@p#MOIFHr6aLIZ(X{^ zBN_@DM5RbJ_ayN#L|Zw9kZ}nZhzGRwZ8q5wfFs?>4I znQWIf(KWer5dZ5~A-((a1THT&$gDpW|2M+T=ko2GT?%X8#1Bn#V!-#%AEY|E{&pw{ z!jrTVXCC5!9M=@NvoHFxpTF?sgems&8Z^h*zHr<1ygjL)1|~BP^hd2x!6Z44T;klu znjgMJ9ga#&WyIdN1;9T~Dbcc^MZZqykuOwy{Tq-#MTOi%w(9AG)8NWXW7|3$@&W3} zPx-O^N?-->4LcS$p)6tkp^%&9^AA-fM#jRq6!hxP%_dS#3Qn-^`zBA0d1}<7(Jp`_ zAs9x@@TVY*E7Z(K5kv-a%E965-+zHs9W54;5s8Gokd|A>z)j6pL^+@guSBCzjk1Oe zVeT-mMbpAGzQ>zz>tX@)_1fZf`$4xytu2y+S?eu$t}lU>sx_sqXK2c;X8~*c*qt>s zQJU?-&6~_@%Pn-mu!O*2g>a#7%Z0D>$~xvebeXD$v0p77MFG4fiz?I`@_lUWq;F#u zG4C(wAF*8J28<;2&*@hedvO6juRRo>l8}_K~Na9F}AbpC^AWl4PF7fb^

^UT zgT+1Wy;A=gg+G593bP_Ba`^xfj2jCk2>P>(AfIGHHCPD2nD^q;9NJVFtZT+5`~Vw8Zhj7xeR-h z+{b8;9u?KSOJs3e;f=R0m2z)_>Pf$RR=DH)tdjn>`c?Yjj^ZO|Bd_wWp9n?v%ddmZ zyqty}DimXNPQfH>w05{x5pPh-Pq|M`=r%N(qU95tX);T7m=`VC|EP5p*XR&uMxH%0lT8>>AKJjFBj z&7QEb@0B6wxOo8Sp|KZ+iYS&$LN+9la_yEcdi)5 zqKGVVoi1{j+-OY5tvtUd_VI6^#c~z0_@#@}qlmjQK(7lZZq^&j$GDTLFi2}d^v^he z;$q1a^xS{ChX<_CH-yC8CC07DC4mImYbEAY(C@g1Pgsc53NNB7w@$*md&H=*3S|?? z>AY8z_bwr#{maw4lXA?My+)4?P`!c)eX+kJ??d!vD#uH4caRgEqHSz(EzONL({g6% zi+2zstL%NX$9oki>zCp0IQHA10F5&*?Y|S12io4MZK_YYdzyQAx5fkjK#a>fj43k` zg3>}27TS_7d;gmtE*L}|Cbi48X;*G!8m_>uFfA5O4mVeL(IVo)JK^|a0N>6qE5hU$ z>jn;bWV8@*FO+H=hmnwJPZzE*0^QE3EqNpyVKK2YH2)n22h1+xtPt_sJBNfkyckvR zC)$gemahq(dOgZ-2SB=l(oA$PpLhhy%B!|dWH2`HH1;n@kq~K;;uTwQrN;TwHlnRx_nZC38qdsqZyg0<+86*-XMPLg{E*=JO$VAljUQlWb!*IBHzhaV2kP z)NEImWdfc^5B+TS{ShZgg)BN5hE{tL}={If8c{yskPD+pR zyk{rn4Qx3L>wb^r&)O(Q^v_BSa!NU6X1X5(MWzS#Bb#tjVAe(?xc8U6T*u&>3-5GT(&?k$k>oOsOgTXrmfv56#@ zU`v~zhUZOYPkc{ePjYV{{Y6W>x}UDtf6EOEO_4dr$%cV_j&|g{;KRJo_&1YxTO$?# z!H?2ijy;rBM~M4sm$+)Wf6<2M$wLIIdQIdw#bUaWTsQqU)D!Go3fswmSc)ZCl~0N) z1)W~D7Eu$K@e@`?JCD;*&(P7peR~b<_Z{2=x*v_i(+CZDBAcBDa-)kiBwygO0hG}} z+$=DhQDnMu$Z3bh={@BGPDXDT4h{To%|-S75^d|Ko6#TjwQ2QImFB|NX_r`GL8+(1 zNkgY|n7&6lk=^Y@X4e#sy)uB^c+W1zS?X%^t{fupdT_^m*HyG-Rh7}A7AE!5&&N&l z8I>kr1?`eTG!ylzzgbseT&s%IGtMvXsIyB`-f@bXc*zw4D~Y;1*{PKg{)zn6;(YzP zj~$6bzF5At_kr1$N%P(Akd(#gyoRy~Vj{`VdAlzif6YkFM__*YvBS=EYZ{}Y;Z;L> z#?)MeAvS?DTeR*E=1p?TADW%l_a>EVT+#bz&ZU9 z;K9(n_z@z6b!07nvX>6PCAN=ic395}^cd}#Ri~J;@yR-hcg(BeFQLNwoHU;(pBjHs z_E7Rrt=pY*BtX%;{qhXwn`KbD$v46oV4Vy?IW9sd%8AN_!sZlX$fr13TdAv-YD`7c zqe9wA$feY4^Eg|98#8}Rl61{=KUt`ESZ}X5h?}|3c+Y#L`ABVBG;ecaN-%4bJ$*0e zte;IaFHGHUOMxS2k+i=>_o2%@qoJPk)wyHuYD_ChbcC9U71=czH@c|=7uo?`j|7xr z4M&LR^>o?%Yx0L?<+b2n-Kz=6?qqH*;-s;Ho9Cp~k8-0rVy8BCT35yaXNHzmj|krW z7Bq8Q?fEuk4cumi1OStYauY1${Fm>UQers0QsCm|X$r!I?=noQPm<`Yzjc?so?-?W zeU=-LDB81pplTkY|I`@(iXUEcp8CUB()?_hp6FPJTHLf&-DaBK-q;NXXog&@7}XS3 zn5t9xJ=_xZVJk7DGTGoJ_0YM@QB!QHm3>T^rK``2 zXrk!p1koKWC7B`S=kjK{OIzz0Ib5c2mGTBLl;*(?3=|u5GUKv7+zc1w@hRxPKGalH zcC@$>udG{Ugn>RlDcNA*SA(NvjS$KFK9Wf-!Nc6_PnsjfEhQSbT!q_>rA@HVOxSr2 zLa&(jtW&h_JkvA>)UR64I#gNd|Ka7jQX;n@JL>Wb@{{*fO{&%ltSMqGrZ)Xa`!Hev z3^=Q*-ksFHo7WIw?1?7LlEdKyQ!UwBH!67rtyO?N(xi$v2rKeN!ScDJigrh9Q-M|R zQNHb9($+9g&Sa(T?d{FDhr&Nce-H0>G3i+`!t1=T+n*P+E?&?88Ck;6q%m-JTMkSIgx>AqF z=PG!iP?{G)*vDuk33Y=a0crQxW8HY@Gk^ZNH>lytLlP81B2m_I9+^vh{+gM3eg)1H zZc+D9%Fi?bJbrRsy0abgtl`1aV8HRLSZHzO$Z|6jBO>B_j8?4Q%gCBjfb|`oP^z48C?dS~6 z3pAKG!r8@fcuYE70Ix83_sK*^<``9*9VD&>vmgZnw_32CiYobWa^NnkG~Xy|m(O>) z+smxZ^@e0yScQ?E0E)PU0l*sG(4JAUhc$g0nU0UO6n>Ua@`KI5ZY|n8Qhm|yKB|}I z1BejJF#l#_SM1#h9492b!T#Pd`5|T@Ig0fW=Bz(?^V6X>acz_bAsa(yc{#Wl16}fWnX~B@e{My4AG4>EmLhhz9UU)Q z=J%eyT=@fQKdVRE)qk3LuoLkSGZdTDb$aS)tBD3Y{F~*1T zFw9?DxykS;Bnh6cBxM0z7v8w|_qTnU@9({T>XZ`gmrAcPaW0uL70M4vX#gHQFv#93 z^K*KTsmyuxAJvrn=KLZtZ(r`hroOuIlDJM8AV%9#A5n%Dej6GqEzyfKC~JGF zEf7#Rf$EDc_7n|kvi>&M+R@?_(X<`?V z6|A%SDzPA=O)%aglhhP zR=bSa_XLczN*v*eRP|iavX_cq*H^U>~GlBth(B_t6K$0Ah4Z0x;!+hDfjoV8GCKz zN=fShID0j>={EQ_m_Ufu`?GyfBo+6N*u(}^1<|j(KSNzvqTW|znZ5>(B-j)m?C)&t zRzbHkUl4}Pe-sj)`=L?=s}e@TDWFL+(v*?x`h_A3g0x3|zxK@B(R5wNo!qE=adGtN z;mV^g;(LDFucD~=es<(v_DcVE;NpDp)bshP;BQRBvWfuBW@ttLrEMkBoN0nh`Pn6J zSj)ibhZX#04jbF~{mKX2#0%GvT1l19elM%S^j+sVb8P}JHS1@0KLds^K+}onorfdF z^!^Xi^uh^C)W_mVd(7QcsKLZq*HKtSM=8)F+zQSkuu0{6G!^GbVWrU4(#ciUZG~<# z)869ld@Ndg4ZqBNhii&0nl^PZr}*f}eEsfT1`d2E>$(bp3|xQX*)4&(N*@0BM+yy& zz6l(fivHldogUJONK7R1EVEpO^Rl0`ofQvhg?Un(L_q)@Coj)8)XHKE&|XOyF_I&w zfEYO5oXQJi=*2}O!iNxT9yc&;+(3>D_6rbkw_3wEMYxf??gjgayGI4zS)8?Db+SJ|p zc?o&tuH(!t^osY{@>2&0Kobi2n59*;bN;6vznQWya|cq5ZF76Jbk^$7GMEy?B%4VW zaNsJBBim3>Pn5?4C6UF>AcDBBItA@fgXR^SFtiP|ivgpFo)+i+PdhFa={%1y`-C61 z<+zuo%#zCPB5u3O-s%6Qq<-3q54dIim5U6P{ZS{G#bBQzBhQ?i5@gl~%R5I|Z;oGS z?2ag8{Q);xT{nsh#Tapxsl?!~quZORW_l*qb`7n)%Db@u_%|r$4Y#&&Y*yr40KTJ} z`krNZ^}O~rz*00ja5S+s$zHW2X~jPEIapvKce)^9RoMy-T906Qu8mC3MsPl&oK-Em zs72a7ZiTqHE$bP%M^L}V?R4h`F}HKOba6|r3v0mWR!P&LzM<;m?Cv%Gf#0j3o~m)I z_Tj>R?^|f;tJ9+J%lZIqv&|{JlyEtH(`46o?E|QYvZ3-8r0 zTxbRpHpi ziyFcD@T1v#rLpM8P5cuUHJocNRTLV!Z!RVKH6_?* zkHc*#%(T6TQ0izi`*48I_K-(Z4b8zYOaJ{w<0TR`4V@=cu$`JwOQ1f29vw5NpGfCj zL}B(p$iJRoR`E8jTnbmqCYhJG;Lw3Et2ny>I~d9uYd4C5*h)lCO@0`)WiTYb0X9jq zj#Xu8K2+cv0WDn$qrO$YFsxzEjQ!d=ysa^XBw{QJP9x%NkwF>MN+MmL$`U z+3bYeMw2#8Rl~^w+09nlE0}_c1~;RZ0q`3ai=S^?$ELW)$`FXw`PR%stQ2Wl!kMtn zc|6750_8gV(}n97!xgk!$VN-p_^*H}l~wszcR^S<#so;amUSLn3BT``f0?u1-)yYz z7X8&{sI5v0@~P_5Y^z#9j4Mw)lfc1dG)9T$mEQRqL5=^pOYYr5mEaB3I4P3u-)tn_ z{PJrkg0jwTS>0oaYG-L?NYSKq!FI!!VW}b0Fl=c=HKmQ=wC{;}jVg|iQ(lHqdHD-7d!YP&xorzdSrNiqRZx)0 zUR=77zHu17T`erruygpGAZ~Wb-O+fi#bT;;H8yH&K*$=*B@N{FN08(~A1d!k5YI&` z1$_b@QUL};5@UxpXgWmL{mM=AslUoryw<{JcKSm-w5p_NFssI_O_CeB0m2o21HgaD zb=zXVb8Mlrh1zzj`4f4ZXPnp?Y`v%n;=l?_HA0m26D@I%j5n(_k5eJ{QF8j3FQi3$ zoa@jCNGnH2QZJ+Q7FLePYqPCO0MoTf($6mY{Uj^x^{kb3w(qZg#{I>5Qp8h+fO6PC zG#V!}I`;NdRQ=Q74QfhTVooWA%XqQr1myXxpx=7k)wiDXp-kvD^kKs^fBcC;@ZlA8 z>F(2-+5XBoa^PM)0Tq~dLeo=Cw zIbo3i;YY5C%)Q_U7>obiMFL4$4rnIGi!nsIVao9tltL_5HAAQ(1z^3nN=vBL^Yt0J z=y{Vtl!Z$tK7l2ck51H(FOa{;n1fHoFnb zdR3vb)e!Lnbz&N<+6q{38paKidiE#HZGb$$<~wqcBk&?G*|J zMiY>punT~N6a<|-b6~-!Il58uBAi5jC}~TB*N}p-&_gUPlWiC@rTt4QUU)uM{L<$t z40sb1fD-@6Q_r{kS=3A~+(GmXo4fB9J$Grqht!|m*g8`Lci#d^pl||3@KypRwvj{Q zwW~7_Daf9l_f`^&&KIR!)&;vI1p+Aqc0J~a zwnym)-$1)qd7K$Q{(SDRA;A7r?a644@PLT#M!cqkP*8|A3VKUgkiT-O3rJ@_nwv&C zQq7f0vAIZc+(~^E&=lQp9fc-mg1sFbRK0(U>WWJdAJ8Of1s1#w;s#d3!zkf=w{sK7 zdKp9-L_p#6CP~o$NUMuSx5#Ae#k8DCR~a>kqr7icmRWtIb+TBSxJrWiChpLhP0v9H z=8s-ly0yt_wyURQCV9y*YrAOOShy^<@I_bta)XI}4a#NV$2^wCbSs;u-l zn*Ch|P@rmMv@D?YG1rR>BtsTF%OV*hTz{OXwiWnOJKWNwu^0Ahr<1b{3Wgab3Hy)0 z9%VP?PsmhWo?S6Wr$cE?y7auNSozl>*t^_3s^cML5Z`rd`w8?4^_saam18J6>M|<2 z80PR#ZY%o^8*1t*8X(vE;IO(HMQ}KuwF*%ZlXYotA673KmljHPQbIROK=;T@?je1p zB%8=$$j=@M-xM{)d8aGd^akJaF2_&wm`t5j)8C&%gWSX`m3It#g{^F{MCzRPJSRGGNV$8wiY$M5;G}13=ixp+ zD>*(%?=1J5@bs&0sMBm3!K5Ck1wG@QoiG$oyOS%3E1^^jdwm#p2tYv4^Wdefex*O% z_dKf^(jdpD%8MiPtO#3AJv?-^x1<;8AzGQJby?e)o3*OSte-81#4^wf$eJJ!pzW9t zN4>FSduC+Fbp0!|D1XD)!4%+rB77!S22WYS!h1rf-RWju32(9Nkw|V)J?fB9E*r~9 z^y1y_=$KB3iP6EJ<6b3iGLE}KYZYyOWxtkA5QjiR++EUG)1^T7A$5Dvi^`}utyC2K zCgNy~hHX`q@Va_qdx%Cs%YYp%i}<_b5xj*idl1jy+0rLQYRrBsLvg~(NCUn{@M?T; zm3ChvhapTfTDg9bSx9lX%P90s*r#cGp)! z4Wa>mb4w~L&Qq|#u>sk5Spk2nzWK6vH`xJ@@zu**oh#D*w>VZnwJJH#$%C|~$V=1{ z&r=^I&CVzxK`o})j~sP_!|C3k{K5N01cifWZ;q3T^?*^SS^tM^N{(mv5QNtBZZroN z-^`#Jw6DTYb1zvbves@fb`ecT0(m)!3}UQ&2Hczi^uEfK@T^b#c)M51hRFZq)o^gO z(Um^8ijl=`I3{OlMpTggQ*xJ-lpUhK?xA_KRceq*<&1a_by-sr0a65k1(+AXncsW0re=Q;6x3l zA)5EjcAyG@u{rN){2P;nfI-g_1!vK2oKbQ_kRb1rwc?~RS9)<%{$hqDU{$z)@3d<5Fi5|xOKq^<^L%C%AH`=!-?xDG*%^IA8wGHVv zCx>O7$tUHzpcO!~lFhey?TQIrOA2v+oI9m?S+%dy{`hwWMfv4l?6$v9PD8;Bi=!`x z0vJu;CJ4|0J&VT;|)+(#^+YoPf6KTE*CFyZUVP`k^5;Mu?pzfb_i zZgB3M^Gyw=@a5Vvn4{i0qeM{_hgDEP`uGw$4TG}*Zy3lvi6G{HV{(EF5(bCaAc5fd z)U5+PKKm=aR;{?RzRfJ?`Qan;6N8cF&Cr*ZmxYetOFFbbbI$6+QUgJJyGzgY8V0q` z(CdQTd(tFd0l{D8e$S?u)2ZIf)GFQDg+7_XZMy+D#*)cY^aQaCy_ja zn@@C2UF}~K8uAyH4ghohfe=Dv@@iMSUTT=;v`;n_(h&CqluNVD>ENW5z&^f|3`S6x zvAEewX+weRTu3_w@4^R0xT^uALs_7BQN@BIup;f&$FKNc_U4LQPkYl^0;7XkouQuq zgM(t6WGWBcHC0MP{j;={A)M5km0>6DoxO*!n7&kYaf_UN$_i8S^8P8!|FikGac1Wu zqXVebm_a}t6B;A_aw^Q_mQdpOUz!b!M^lqJhGjc#2f2>-6d!|>rI+H7?3NoK!N8>6 z>80clM9!nlk#1w5@<*k|Xu`kn$TE(zeZbJbfGH{1OH*8o6ALUMEr_5&oj*Nr&op3) z9gP5Nie^)qkrVP^+J1^sy50?X(dR6$9KF8&fh=^ts-5Y#eO42M{>3tU)Z+@K(5Hs* zMr~cPQy!Zl@y_Dck5DZtM-=5G&MGMhUWL6QkgeQOvcCy7aj=+X?y4MM_xbLGPu0Fr zQQ7DQlN+NNk91gcK~4lY6<@YSBdpZPzZ zGFm>tYfE9=$JJ7(K_Tsrcc<5RfrB&KOpyI3O{q!`vs-|m0aPDND@C+I{ z3$iFy)niH$%Fj~$wID8|tv$0)Sdfd}He8r7fvgX{VlLnlIT9AKbsT7$vBY^=OzO?^ zVwa(gpzNMaijJP^SZ$&T$O|!hZNm7oCmlrsBXzk_7ytka=E7jbn?>plWzaYaG??7P z#wkHD{P4t30bjGxW%6TH1k4qr=?w&$chr{)h0$6ja^4N4<{}`?B`7W z6P#^Qf8+Ms_B?%yoQV%22wI+>o2u)K@FW7dr6nS`wLjC0=!g3EX%2h3?iMzj4I24(VN4A%XiN&9Mm%{*l6+Om#^t3r zVCZ8=v%cM%!1`RGkQ2tj=z)){7kz4vkfL#d2WhR{BN|mqX2F@w^PEN~he4VB$&n8P zZV|F;$#=H_hc$Z@b|on(v9OSk{)ejQLjzTj@}!G~P=H<8e-H*&LOk2|y_o#d1A#W* zgXNRG&u-Vlx+5fLZT_P|K)_2D9QeLGFqz@Q^ib4g-0zdN#e2gp`lTb_==PHKm<{S?6DVeppkej~1>i2xGUJEBUL?&l-MF7X6=4dah8%|b-PX1w$tIsa z>}<9$F#JanI1R{`{MQg7V(X=!rO!BNfQhZi$Cb4(fEjU%1S;f*Nee?GsjOw--jZ8gy$=uqc@{D%>- zI;aeze&Ebtv@{jHwbje|BuwB}_dV@JH8ndCm@tjXxF5;4nSN&hoNdz2x9YVZ8fy?w zwhfC={ogm!P?02TP3Fa%RW5HfMUfmwij^X+sw3@Y49XR1%K=^Dr3?o`qw1EiQuiB0 z<{oZ4`ui6kk#8G}}t=Oe+IGJ%dnpE%@>S%}} zkaCDeuPi?9%k`YWR+Y?MMH62+3*JaUOtmDfzWG`BT-I&{85Q8pdcZj94eLS*Akn}% zCaE6RVPbdaslS3<_WGRfloJjVc=M$Zi2?3fRz{o)`&@ppOTu&b0%Kl^eH|SKYCvR z6EsN2iK&?G^#~SW{xX}$z$=Su*P;Y$wXv}94U)eTGpj~|aLOpDG%ER|E&hjY7W9t> z2bpwIoInVDr<*XJdKKOlF_oG#-`T4E=lQwy*fr((q^4&^Xm`3Fhl3^oIQOtoYlavv z0$S(a@xYIk8n3C2FT5}$Z~r^Ob*sh!Wi?|aji-RXj<4HRV0i8P2Y+$*TO4Fv2z8_V zubGoPQMbFJs<9Kra}t2Zs~a>)UKHT8RM<8$HmO(aE7k#`s3Dqtn)6UEGHNhb8EL)U z{a|_RGm~D#gcY*=oqf|2&i*~}h;kv4k^~f>n9?~r%;;wA`zU^Nu{02CsW;fe(kexh zZ#e`0X?mf6v?6x=K=%229m~6*fsXg5A>+E#56Qh4%x+^teiC6RvSJ_*X1uL0m$3sw_} z&64*O_;JnyKBEK#c?oWR7@&%>$C7J`j&t3kOnSNHmZD~fb`m1G;vMkBV)Zx6;bXWx zTuwglSjQOQZI+`hp&N#$Cci89mk=-DDN^Pe4OXPNlpD}DL*}8ND&o%xYbqv@OI?a} zZa_DYg@_IhCSQU1N50Rw#kAEYxb;9!-z+x+%V4pfo-)^jURrC<7lmJ_pFcp0OSCY& zZtwkfDV$JX)aZGL0aSR?;1A}*NOD?T<`j29FMEM8yIMpcd*{hZQ3I*Rsxv(L}*jn(x<*j}Svh!+jtcx z&{~jZLE>1TID157y*OlVJjdk{@lq9^`8OkLv6ZEIQ~LXv92FumX)As>p!`-ZqD4K& zHyL@+zpuEi9pk5{L)JQMo%}A!Wu$C`vd3jwCDbF<=%(IL_LRsnvRlF(doik=-k+$Z zA=3dN1F(pe8{fRb?J7(!NMHnu zoS1v=w=Jd{JzlAsSqNk(EQS&Yq)%xW{`NmR^Pl^|fq@1}vPGf=#p=s3M>IE4tLm81 zaK<6G2aDD;{IK9c&AB&Q5jGdMw*6N<8KR`_a6d`I;&*bP4tPJJcVNDT@XaU*$Nz=# z8|&olCu?dl(pyWj6gXeMm={P@`Akg?!k9A1jUwII(Jz*0j=UK9f?vo6V_xlG`IlB0 zzsjR;oVkG?B`84xNMbm%>O&swfMSDUZaSP1D}A*Zzv7D{0_=?1@gcQgg->BhK*V`O zHny`MzZe#jf4A-bngWD{6cm+XZ*z%71ofmFvnMZ*l!`#60llY~Q*zz8`Sac02C+cSykPaOvos z5ln=5m;ROi&U*1cNb9~RrdSAYE}RVz9>%p2*rkirB>?_RN-ol(yS)n=+Dv1VyISy~ z{OE9R!z89F17y5xhh zND{XgLfR<@waYpC_scNZ+0q_fRyTvyma}&M>!&X_AvcrYe?x{o^ip{@Ui}V%x7&U1 zU)oU-`jsGLQ*=_^`>e-ok@2SNxAt~Qu`2d)RSGZ?^bsg(^9qzIM%64HZ+AVy?4hJi zZpDa7;qurlpCOIN=N-0vL{>{Qqx#L6OBA8+jE=vVGcp*3LXG7_2_uU+GbOOHlI>)4 zPLjWe#mmR|LsRZjBR-PD`usz94!1Nd4%2_br4)&u4VZH3&QkKr;x+P8Y@f$shNcv= zuX^#e#sv3DFn#u#KaEF{cYe*{c;l66LHT!*P*72qQ<7E82$VX)*4-bvENWZq9(6uN zv>UvYp7btuhbLpiMgim_%)XejouSOg=9#!OuI1(;##NYdeUYQI5{8)B(eQzc~qxZhjJPV?)`Dc6Q6DeR*@zR&X-3*!fC)E+}8Ym0>OQWpi@Z+*6eu~Ii>_4qo5`YwOoV4 zA?tZ2g&|QM=l3$_-G2aUbPAD>T>pI{I|vK;-e7x40!irUVrY35qGmPpI>K9Mwq3+X zREl5Y_#SyZ|Bor+l@t}%)Mc66R7D1dZB0X!AQ$y>7#70}Cq$INp6&))2(`&$p&-oO zWmI@T-YEz2aY&GJ0-+!}t#ROL@cVWMJ7!t%bE<#n6+8E+!`+0(OF~2j5rX0V&#nD) z3prs>QNvog#eXMWb-&M12MAJ=y?t>=D2`#EhU2zK=$U#ucD?x>Nt2~1;x_iAFj#=){`-dkOfc?PV1Sb5h%N8lX62`OT1%0gc4yzrTUiH(n%nED+QvZ9C+@NF zam9Hw0Xl8(>WbvKMrz#+}Zl*#h3J!<`Op+t&+ zTDE<92-9IXG^HTotRF1Sr2e0kD#r~bM3y7(mj-&nNl}&gVaKBV|L1fDyF4|DrJqwI z3|@Gbl1lAEYRv1QtoQ0>_MF4tdzB}DD@o!qC1Emk@EBo*Y|L~65-4|4-?2M7H}DTX z3r=)~@G9{Odnb?E7zFRktC9(dvRj71aZE&o*%_qht3&RS$kMz$EW37mEu0s%J}ij} znm4wPSPG3pUD&4jXYgYh2t_vuFki3H)0AnHm8&{To%nwa6V`qYifQpuL|W4W-@vWN_c_xy}1Qm@9Mf zMCtXk)C=VwHQ8exJ5oV9JlrB%vGL(0J-k$f(5h)`8DGjQCOzQT@**UbZ9ntXIkf2` z={jjNTQ2jdc$z7y^irte+bE?A1H>P26G8Xag$?3yu2D;j(omxC@5Snxss8ssjf9A$ zM6E$(gZAtYQ5#wRKc>DiD$Zr;8g~MNyA19cAh-+`oCJ4w4esu)!QI_0xCe*eP9V6u zf0J|Xx$pPuVf9+GXsNE=Rkf=sm!w7Cp&Rk`6bEyv7Tk1U-ZHm7s zQg;*i=Sy$)+7zN=z}kmZaqBga0;;R4v_rPlZa0J7yEzGhEcfywAn>_I?@1AEKfF)E zcq}E?%~KBHWVwcv?_tZ~L7M&SdFfT-gunoe9WMBPg6J=7^Y_AY84uk5cSrx)e~l1O zWWdxw560k|0w_8uI18yn8F8GQ9}>4GO|XnSJ0BJs(0t1$Q9$Yj=iu(AW_sfs=UJa3 zi%Lpbgh>V%Z2m(@80j@yo)UT=nElFep7*7%C6n#vL7#6;bl6R;_CmP3ef0N|72yb@ zJ~I3MXzuIA%5`d{*Aufd*+>UEn}m64&p0?_B+od+FWSJrQt%HL*GC8&)d|23lZyG@ zyAK1N>b?v?2I{IMh+kE0itADiRXM(;v*L~QkL}dQig%?yzXcET!5PwnGHSJT@Whm4 zHX<<1w(ET{{4a{lUXSd5TS#HiM zFcMMxesa8bJWsQ3TbQ?}OErZFa?MtwIEBc{s}^KHv)`Cvdenc56{LY1B^pn(cXu%-Hz~6Qd3ja zm%k}mMnpy~*GZBNrhrTQg^y^^NZ9gN#?7RN>C1rHrC>AMMgACzA*B#4yDdZHrJUwk zCQNI{$;cR?@jtLC{Ej6|a<5@b;J!iw3Dbjk0^E|OSzrOk4{%7*5)owR559NR(83W% zFWuZ6tJRJ@GfH$^5vbUFOSOiQ(NvbQoG!0czd%M|0`HWToZ^^Y#)W2<2wg<1m9Ve? zio%eG_V#NICaX#B=bjB6uJEqI$(egN|z zh{U+)o-Hx(-~SyN`3RHlFx5;zRb|mj?|2kBlnTE=(skLew?ddC^1OCEiJLQ-#xVKa z(X<-zAmi<;np)%+g>1&I7sqS_*i7Bn7c%#2!G2I3lwX(>;n^Z@B`Fmn(Afl_LGPm9 zV!cT#6@&aVwt9|LFdPL4FRbi3((^CjSa-%WO>-k2R7Dmj7SL_LHin}R{O^49VU!VA z5l72=5Llt2ASnKRKE()A+@A!yjnvpAH5Ox6jO-DLzg&0*#;#vEZKclNI;yyL`|Q;C z`pWV@zeEPqTYNk8fe8!aKTS#;s5cq=TA%*@!sDYOjh94$4Fs-_e>Yf4Nu~#~Ly0D= z*LS>&Iz-B$^Eg~d?_4p|C|A$>X66a;-}8aI3ul**W>w?Ktk6ffLMoSR%NMDeE|C9k z>+~@aod36v`Y=~8kgaAcex5E^4vVwWRxA40kJGCw32Z{huQ%&8wgnEt58i;;9CIY3KG%gnc;N!z(SmKgg^x^xJ@b?!R)|yP#xJaQvGlZw(EbVLUs?($6&v&0o=#^d&c@O z>NEoAZrl0YaJHIiRZIw>{a2~3Yjhj$7e;Jve$X6E*CO|1wZ@l*mx0LL zEHmj1`-+~kHIwYaWhu){9iGe(Ewj`xLow@O{IBERKJ1A6#Lj5^MDxP+a6XPpB_)+l zHj1ic#4v)w?DfCnYYYI3T$NuxDcirt86{i|oMx*S**CK~cIl*ZTl`H5`C*5uQ8Zl4@gY|?TPKElUHFbZXKdbUJ_3UkRs?4A6d;`(M<%yA&8$uok3H;w;uNx*{Wzb2V$q#94TXcBvPPU>KH@DyBe>w0hti>pg1 znDJaBhG-o(60)%~i;A%3{>(D?!(A`TZ#(Jb@W+FD8Z}{uiItQtCJwleW6;H?DIE^7 zOA$44`-S^S0Lu@~2|8xxl-f*PxAFIoGaVf!jPlM#!sg(`4`Gr(vfxr226W_v18L^} z%0Bor4TR#4w*nBn{r|EC?H{8dKB+}@^x+4^Me1s{*iJfcTYjah-Q#fRJEN}ewM{ZC zw~+DFxh{30w8JfSY9jh1b6}>Jk4M0#&(cXQV=1*hP9VtAEFdTXC2}TdAf$@?`ni1H z_8d?w5~p&L^M0iUK!MwH&1h6W>lApXu;VI?TC9epjE@^lB5bB;_6u5r3eOKy8^f>r zc0cPl^^bnhhjD}fZ|1jMQG4$H4uC(o90-jw^a+NoX}OJIQc}tA8`C)FZ*$XA8!yW5 zEf!--knkjwf)D46`)8g$RQia?g3F?W`BGh?_ z&Sksm8Eo0VB1%R-V?>f`|9{g{f%zU}jZO0Q|J|1>WPjC4pmQC*u_{wD4_Cxl3OU6w zMql<-Xb(flJF`>&Eb0oQ+}tj#vOVIn8 zsA&PR+H`~i1+Y-cYFdi2#;GU#KL$Rn?t$xNMZlB1bh=%i;!jq0oKo);S3Yg3-FQ+9 z>GP*bv8ZxVlffcL`PYHMqV0RCF)Ef(`OH|DU8DXZwq*P0;9I`WjOZHnuN>>c@P-N_ ztHz5~!rrR;w{r*jRSu`XWF?aW%O=tyL3%*#vxqoj?9 zkoAslhf}uLddv?iZ3pnU73br6MN{AH%??8N^w@EX=^d&_!5uk4TUL%S`#TwTiao$uf4n3CQc2xIUaIW-9jdCqM7ywQAbEXqO356_Wwl3znQFu(pXDz=iC1Y6PW@4EQXUX z;fk;ry-@CC!V;Q>&|se!m6p@Y##e^%L{gL=_6ww=h-svtg*&MFfJjW;Jdvqji ztEcJd1eXVT;3^L#(GBwBH%Vbq@C&iLG^L~ZeekEEe(W1_;O+$^uNmqWw}$&P+#8H> zVV{2u>>f#)5ul-TlRF*T>CWMp(A!wQN~z8fNT3n0SVs}w<|lv~Mk)V)Dj6BFwZY!t z|FoSo8AQV8>68kp)aQlU?*M#h+r*e?l(Cd0$YH>MBXZz(PH{xqY9P8!@PQ;=P#|PI zBrawn^v{gz!6EK_`12f(h_+!g8g<;8b0Ph$91&A(`)%=^kmDLmWL!=+TbE9v4cqN_ zp;u{b&-GjRcV(NFZyy@rh2G+uHFe_h_T0%XCYnh2wJeBE59s9TE&UN zZlZ^yH0zT$8uk45sm|2_LyBeQ)(7MNjhMjDW?Y%5X)j+PzDR!{6J}nK$6)T{ItUIr z2<|p$RY}jsOL6HG7D@`R4)b$E*g_Ey_oHKXJVGA&vDs;|_OKY4KqE_JljC?n9vUL_ za)7xX+wCj0@$Eh34fviJ=(+^U7fo=E{}<@}eZySH!P42WwUhYo z2&4qrJ77dzT$8L`7Ciy$oC(g795SCA@+uE+7n({KG4P4qzS4*SL)Nw$bHSn;?T6Fu z*-qeQcH+ZZtz+GLvYH-+Jl}5=hwEq|087_}*IWNwKgP>%J6~*&XbkZV(N{dIcgeId zL{Dc}r-6_{E6m}pOv8Jn0D_p&#s1Vv$#fQ4TJ(<7<_^4$VG_d^Ncafo+GxB__0oE> zS`uou3D+CCxv%%x7DM}3hIyliApTj2ZpY6`;&-uHT0#;c64l^LOidy7!*v*8pWc7f zNFy5f!C4K(&6)Uj#2VNC(^#WgjT_^`nv23o9%eJB#RL63&*#Rg0iV5ooq1)h8zb5C zmH(mV)PAPRE#G-2S0wQH{aP$pM4up>^POw*2xORF5GdoO)+ne@MsqKrksqTr?_4oMS;o3OfWU z!y!AmEQdSiQWuiQGeR!?K;6;O8^8HKh`bUG234mpQ3_`NUgG9Z2*W5U4t&)XSI25E z`DUM0!Semj=k2Ye#I7@4sLU(fp^{ zyn=gv6noPKW7FrVs`(*2HfoS5rqg1F&F1!i7K#t@i4BgI4~{89B*Rife4+0$y^2&z zEL+Iv-<9%)f!9t96VpBP?}0bs```4lEZgjTChwbRg2<;pez)HBxn)aBtc$R2iM$Kp zjpyI)+G;2{v+{MjKm3CRmmECwvXj~pNoST#Ro!)jba_bA;J8=5FDI1kIY~txdW-ET zi9%jL?F3G2c19P6M3*GzF@fKbC*5z2({a^!zvu3@$($}aCa^3rZT&iYly4}e@>R7b z%cH;DvdyR05?oN}ap>o=44|Q|Xx445$cW&aKuci7eqX`Q8jFnhH@JDwQo<6wiW^)a zcy&-IoBumU-k86u8;vo9TNQNavH3B3A7&(f6-YST>fcjb@W_v9dg-wn z8+Yh5=HygWdHT3qoC)dZ6FUxd{c$glh-G&79FpCiisfgt%kju4Ei1beXt#mgO4J~Y zoJdXvgT?cF2AbuLw6hAdFL znuFPf;j*FxPIQPF3@k6*X2XGJW{e?ERz|?~Jk5p5vW4Tvr)sn}wIee)B!omU>+Hq}_T5lKQyV-zE@!9blL4hxjbaJ3VUPJw~PZ(!YJ z9@JxZ<|(XU$L$e`w(8&0>V0oByzFJ!MFG}lU3wyCLjPEWQNm|H;n<$)TU3DuiMm09 z4)I@l)#&jztt8@hTnsX~snZRW0$yMG;(jp4d)nLcKN&TV zV4}Vxk-*9tm~INEgOTuC)Mb%M@stNdoWJ($xMeqCnxvZJ@;hh9nBrr{$L#~=>(-cO z5bqJ*^2lj=sT7b*PMn8JU0`-c23ydzp;Ame=6NYvEiuCohCi^U&?p9*@2s`JY}Y^p z&%YW3#z)nl)ML~C$o>U46f>f9AqBvjux@@Uj3nw?D7YO3>CvX8c)dw4hH`vt`gbTg z2I0lz-!nov$l~FVMv1vBqCMg0^_LNu0dzoe;(i@GPtK;M`%jw`)7G`j+(?FsP!zpVz4n)&vIMqg^T>+t>c`Y>1ZY#o2Q!KZ86+A)$r@nE$P)cP4-vSc%im!UyzE*{6f`*6XOM0 z{{X~XXPEX?%OAHJyTc`oO9#||0*Z8@XMP7mAMNJ5n*`~RyWGKj_J2)#Vi93zVo8QJ zu$_n?#UhB%kCdQYJ8G*K!G7{Ax&w96DK3iPebR2&!Mwz)YqJ*-&5_OocDFEjUnNdY zy?JZw+(1h%C63ZxO(S9sn54c*{opW?XxO}b&S;3sg`+IQ;~po0Wc)+l$4!i@97blY zF9t3TMqGN9ut`PkNk1^qY)mn_4&0U^^>C=9YHJ1C6~dDg(}0Rg2?HHR-LC2*GT7!k zaT>AUp76e6fwCVi?d1a1EvsOMN&PDMKHUF+QY>(5$PdNv3Qq~9Fh&62w2@Cv$V>uA zz{0F2Mnlc840b=M2~^N^2oiWMNlZ*dCE_ukj%4XRdbzwu8`WELsR$eXje2_>|DEIc zi^F8wet6c;!4VXq!`3*gv_$RQYk@O!iP|+jDYN0pn<$?#0c^v3mHK!OaE1IoyqAz* z^>)Jub=pMWYKDPzG^q2Z@HtZt*+9~vl}RbI3hqse?UEqIE$Ti>!ifbD|6`%l$}FPv zp}Oc#7uglQV4f8R>&8R4nDehHFMG@&zuN-j-iexW;sW^LuvTi=Lm;k1E;K?R$EeFz z#l{0tya))XZ6Wpyl%AQ~smaS0ng3mBYVyPD%_sxr{#jF?z+pW@HC8B*%lFoc=lyWI z6vj(>!_yS2yONyp^BV@25e&!GfXCftyDPqB(cERWXcBZC+%^&;#`ToGW?%L`G<1a6pvl`g*RZccD z7nOdaFfG5uminW*_(6IBk8On&O)Rz2A2KW_NjN4MIKvRtFJJxP^Ob%v+f+`Uwk0hK z;V$Zz8^4YS*1D@w+(2|ozi=TJ?4_dw2K{nwaw8kJC7`H=O*MA9C|F|8UnV)A3b)kT0R*z*>keKf-#N8!dd6{$wFt z_O+a$RQTzVuQbQn_2X&}C6~VViN0M0zoNT#8&m$dy5XM=GQ48tW1Si3*m615fj;N7 z@)1d;83oM2N>UAxkMHyr(}5%Covw;73GKunzb3z5lvN~K*S2^RL^2HkcMzD-&At`1 zGbm!b$cSV36YC4V08(r#GMgYPewy#mpui+H%m-qV`_BFGfvmRKoU^0hV$HT2p$5?w z)0J;bBkU}5Xn0BXd-H8W6TaP)4tu%v0W-00r=*6P-uIX-FDIr3>6~cycG8p>hL`!` zjxFo1xROy^u5PY%+X-zN2r#JdGph`Gw4-+!}RHbQTNFl~R68Rc8Zs0Nbf@hVtY zT-C`2LFd`C;&C@hNf8Wc3S(*NKM?as$;1WWEaT5`vX2esk3gzZxe{rmO4A?&j)XNv ziPHUcsZrFfI9lbSi{V?unnAQ+PhiTf_d-ptu`REU!Y0dl(z=!K+7A#e<-wAb!Z&?2 zCtaMp*v_9mn$`F`Q7oh|`rKED( z@Dk^4h=2s>vEE0OG9qAZFWHqk_nCS-a=N)3p7J_)OWq!?7fHJ#Nf9eWy_de&&ntnFsZ; z0T!GD#-Hk#^hvzU2n7(1B%~M?D5Uu?dX@D7uc}4%eX}VcRlUR=R>KI0{*EYoo|-0*beBkJwr2|`NT=bhfp!5&W4h=Hcd+~UB1C?h@Qqh8saw?0*j z<*%`(Mxv$-kkyg@IQ_aQwp!d)Ry#I0864X*uB`T8oL2i8(1cuRx|R>!Le$a4CWPxy zFcL}GoI7drErN@gfKnh#|4{&A)(PtxqnO)P=yp7_h^&joOi}f6x>0U;tpw^JBn4%n z!7$=bkN-lbbI0VN(PC;cZ{5V+o|N<`rP!i&6r5vwB|lLBsx?og73;@Eq{MAwN`y`X zDFGB5yb7U8Qk`$8j(H0yx5A$r$c)tu$f03by%}qr{T-l;V0`*@cqOLsjJ4pR>iR#Z zD_lT8@()RHYDok_0)m5WvuY<+c8K9XR+%8#ivV$ET8w*tzzDbrPaVlOFtxI}e0Z%C z>B|u%OpljTl3~7%-#uzvf_f5cQ;6gBmmxE~s`=u#qw#_0T8NYDBSO1yX-Q&cdM@kS zSZ}oy=2z9xwUQB%+z;BBBcUMy_wXVj1lCk}aoa-wg=LRn!W@^4ENS{^9k;u?CXcl8 zlFu(sxGRk^B*;G&cb|h$aTUnps4yHYho0gQs40rYjs10L3pI9^H0z31gVT#bi~a-= ztqy^-;H=?qC==%Vv`5;pjh~R>KNgzsL2(w^t`f8vQJiI9PJ_sRL<}ENr=ALH_1%jW z(_?`6N7$pRu&VJ2Us!*Yfnr_A#naEqO78JEi-F7sSYkmsj+)e&Q0vo?80{)p=pF_= zv&cOhur{pywK}MNrC?71e+lO5JT&?&tU47!Jc)V82LXE&)NUPQlI7**B>932e2TDx z#&HP#1`((wbF~lykdD3u4h{orwzPoph@uS-K{bcuvXXAC6vfD2&zboLf1AqTS;!+Q z>pLhzWK$ex`nxX>NtCJ^_9yLx20sI)l05H>&aD{K)Iv($t`i=r`a{vq-+kXbv@^l{ zkUr&0x^%1gj&twl3GUCWx?xH2@d@_8os?NJr6`Ab%IySam}c@{4I5BuBmXGyH;QR& zBNK}}gk&5Zu`AN7W*J3QkjP`38?panLfixfvV<_srx=YBsVaono3P%nP&e|x?@0`a znv^B+DP@dg8C1AW(6{S#@%;5*>ZBQ6ub2bCX z&}05^LM>VVI#Sj5c-CD2E~CGl5=sRKR&*i8?D+1M&A}p zC_3K^a2-PqA)@<-!B}XFXu%pPQWYPXH`qMuriZ~mM$Gi)<28Yx)ZOZX{i|^tIn|f- zz2ai<6AYNlsj?s{f$qKeHWybKVEK%b5FQMJJS_DOiwf6_i@znCkySL4wx)V0M0jQ$ zB6YP55ji-8HJmevh4r~@iA;1QahvXiRnvBFa=UQ7FR{9r@L8@e`n+H-d~@`LGD@zg zQE{!bHW@_0`taYlM{WmT$@G#fl32aTvZ(cBmeizfXszO?{n!^6!?QE8c8qslDuaH2 zg1F3HM`lu&2OE(jQ*;`XyjM``v<_7ES0*+{N9AI(S?A2Pbv~$fkBvH2=`eigdazcf zP#Q(4bGZ1(8S&Vr6&6>xcv$}dey&c%`+7~6B_*mfZPQg&t{qjz8@V{ue7U~Y@3TFT zGW*E1q~gtO;#)$FFAQYw(atA3j6#&DQb z21SM?1t-js*mkj*gfdlY;5G0Iqej8u>OrVRC!=!9#Y~;*`ct0mkCHoQZ(Y#M{NwIq3R0>U;B*>ExT+JY6d8Nro7fjAk<~5Ay}a%{GsPUk<@W z%+Is5Cpoz;VWeagGAKM-#BB2V<9%*^!IV4lj@^r4_nJk%E`T~2o%pV*yBC>bah$z8=XP>(IXdR! zF;+GCJN8}1wzBks-D`D3RYQX=f2K4{pXqul-sL>?kLBBq%qASZPvC!IKt0C@pyPra zuv720@%w(ZYgfsh(RMm;8k`~KY$y_-KTBIp+Q;%a&IZ8J9g zhxH0%hbq&Gr0WU9h@BhQr&cSnSio;`7VY zTDv}?0|6QSG}240h(x|nZIqKRJI?_H#^C|Zl?#PPaG9ij25zE>8@oO*LD9<9qMQ`a zFwzVx3i*?SG)4!x5Zr9kzXEOhcroW{^hP(Po`-K6033@NO`r1RroLMzgilwI5$69? zSYh$jwoYvM61;SkIvK;{>mH%RyA!!JU0{ZN>b=e{pqSfgyAV4h<^0t8>H#q(8S(y| zgr?FwS&8qytJ9ESQxII~o()ML;K`|r=W+u27?)i)YO z`Y4ao!>$Ey&oATOESkwALd7Fj|&sSvI;mZRQ8WtLc5KYCT=h`hm8iiczi6sTqa$NjZwZ z`rZJ;k-@(GV$a374SDrX`Ty)ri5^b`r=TEfzB)4fc@T55u@20UUsy22NRghG$FNTc zv-jmgy&Ja4C(0{H0CK6iTr(f^xAZ)+mJNfCW5e3@7c&BrS~qUwC&F7u3bE zzo-Epc^L&4P5j)4n*z4Oq|$jSp~t<4gL;c$zR`5Y#;h4MB3Dfz0%HfKhNq0q+EdKe zr<@E?KH_`LQOK5eH9J|=1|QCz`L_c3Lw}7HL3NJfW-%iMGcpLxMyv_$g}}S};zMqN zn~}Nz)4jgSkIBerr8*b;ALF-6x|AC?r}hsdgxMfDT?}plREp{9^dZ?3a2eA@M{E9u zlRv>|HU?RH)Ki##cGKc%{tm@nPjEvK?UIxYS4VqM4v`UoCaveWL-E>H5yNHo;{aaB z>8$NoUOL6!hrU9!N||!^gSWAr(CGSHeX~-kXM4t);D0=df9WQ>h)@=j)m7^@@#@n= zO2vo%80z@S3GwK%C?fWK(0s{mAqXQGe_kw4UTE&Re#7etxH#0fVF&vZ&Ta^{Zd&&1 zIjPR;=+CES{y5od_{0D4Z3BLz<4$pa+`6_=8Kn?3BE^6JKaRji9%+3@I~z5^fvW1Y zD=+JSs^ZmOiD6)a4(yh+)>J?MMaR^fAriR_9pr6m*f${F?`<4pIHNe+VR3Q{_7(| zYz0y`JL^XVvj9Iv>86!rkKF2yl@##5d~aJwRru@nS*-r7ai`xTa1r#IGZwH#4{;Iv zHm@vTiyd($p;@WK^~#S0#2vZr8bN4c6VH%=1GU0b6=i)$QrHq%n&@%TO#}%~XDrzt z|11pcJ|!eze0nTQ)T)|^B(+_t)eWET7?Iz^{XuIwIXaLM_sLbdRqS7Pn6NIzbT<6+ z2TWx6s>j@Wcm-0A$B%maP9{nb+M*_>799;_70se9`CGDaa^G@R?*DM$3Vn29ND%pE zy>}&J+b7;U0W#(jvC|OpqcVtiigQqppG&Jea6-*z*f?o}Vdv{Ssv&0qXG8+8y{KnaQ)AkLy^hZ1$$*TJl@L9JKrRU@`MS*_c@5zX{M{cGQo|}^s!WONru~hM(?#g% zMg@kJ6WX{HsH=F#i$Q%fX#gd*3)YuE4d=V?l&5qh*%Ewu$xcM$^}tYRDLox`oRt5l z@xS4_A3?%=VcBH~m5?e40Cjk-)}>~7R=Hs3Ro@MsbKHK*lv9zOZHq2q8&hJ-c^CIa z!{KM4);F_Q`S+Q^C>6@_by~R00!E(rtOQ1&7A4RhHnj@fTwG`bDgi;NFQt-{a1gHQ zZHoq8Mecpu#^vCDyUz(UrB`qK|+uwYAdYLh$gs zm)q|NdF#39ugC(AgkN|O)LuKeyDvNSzWO>2ZxB-yx@RtBH@jVabJ4lkQ8{H!(+6lk z9uMOs{6bI{MFPM=S`Wj0oK)ilE;|6)gOG8)*2Z^27c{eAgq|UOmZ^ zHuVCeN0h~QMW4;LBS~G~(%{o5zPwrkKN^i3NuMpSoBr^#ML`4^#Q!IOMBktfWq!(c zcbKpw%PS?=t*8>-W(HdDFQ154R$?k>&j{OG%5j*h&SE69%0EO_Lrn0ltC>Bd6nooV z{Y*$#9PJWyDLFYKsWOi;%yzC-+ zm2Sb3<8=*b^|sof5kS6|F2~NW>bIDshrSuL zW+|y_o|$sTi_j>)O?2~aJ{JP=RukmuSPLtu`bZFG7$>5%S=ACH&g@%m;D@{{yPsfU zcTy!e^b7|fZq~kG^!Ap~e`41Ea1{BVnx{nJnuyb?`5I7nr@lDFC<&3_r6{vFp$zOe z)E-KZ8)Do1Htc{ONnh0cv>npxY~^y;Zfn+Lsu6%^|L$=dkU_)ZtuY7*W3sZ9j*6dL z8JA{EgMxtEN10qoU6D1~{d1*+lr63^qd3bk5wl{6O~E)>zS41|SZ(UZr)EQmM=lU0 zSn}DM70~i!vWe^6pTvdasK|bbUpWxoQr0{DjvUrgSjeb^U+w9)D?~rwc^p7zhoXZq z4RCA(K@koHMH$E&8V-r<7SeB4E~ZTPf+JYYn-`;-M@N=a7Sr|wQc~$P>_mU7K1ZE}gW8`1DMpgfjf$P?eCwyAl(rCM6_!Xy zHvZ=Z1hA4@znrG53xXLU<4k29&-sM3my+3sp-$r~8yTCAm#AqVDvSK|Tj2I6z0;e6P>$$iq<;P~`c0BQlV6LlqvKY# zJ1;P3q!u>#Ps{HWP6{UFpTY}K=%m#!GtWkkO*ypV843v`J=LU+PQr_%h@Z;o={22 zF)ELTjJMmZPQ?{XS(l{O%>_L0gC-PB!4P zSnu~kWISFo0q_nu@(+ju&}Kyw2*L%5N~8yayE=5F3^>fcXB{8x6aO=U7_F* zBVFj9wMnH-{Vo5&$X_tp}RJ)RPmMpd=hA9jb7z~!ZY>8ZmFkeej z`8s^D&3Sqy89tQ%S^BW(gpfZenb$1&wk&LN)b2+~` z_Sy)POY+`H7JIwBgEcGJBhvsEoX%oZRENOq3t_#=`)J&8Q(*ZNOta!#;R>CtOJ-F> ztRqx^RE12N(LgV_5bJW-`=_PLHwLqY0Acz^E_`35X5;3cG&KdOGXIM@l0xz=0lJo( zAP=)wP3nYd6PUy}n~()SQID(|R~&WeM?wlSYUbS(kGgD20xmDN42JQ0%-?mb7`@Zm zE?LY;GYQ)Dbkv10-fi4xEEbn;Q~&r$y2v4rl-Da7;mXN2L0fNk5K!q(WmK$7X>Iao zPJtOG)G+KI!@0X+dIl-k`vA>SG9ai_QK~_hKyxI%IZwv*D>-uRw=N&A;nA^@tPeJq zNqr|x4jbgIK2BqAloSSH?1?AfHY6h7(Yb=+-z&*it4W*mUJMooj$9QWg0g302ijw) z=f@u>@O+OYS8sEl0twML8E}b~oH9qB)#R-YugxUD;Oy6?N8J5lJnR7+jA4=~>>$gT zh6b>JNeP+({50Y$aYrC)5GQ9KOo;@J_V$oc*&re^FEj%v{fBF*2!g{P48s>ME=hLWxMoX^rnou%Qzw`yvSTPr) z`g)j!t7$45Tzrv70s+Sy)pG|3dt_Af-e120;MR*SKYS#_b^MmKBwLC`#AE-w<++P8 z#>tU@1-E?43oU_5>ZF_6(Hf{N@Yn9Zf*32m2m6S5+Z5oJa!pIkP(yj#A6y9zc}Mj?gk_axq9o@+Cv{EZoSMGCW*DG?a~ zkT?(zwWtpKw!lTNjMeVbXFkJEU%aWMlTQ|ZR;m->K;_UUkeA|sHW+Y1;EgfmYz|oS z?9ZGvQvLA>AKhI;X?N-8BfM_o#d$xWRtEql2+=?c51<;9hOY22uw*d_fxTHo=r85j6DvOs(!x-JDA0Ve^d}!DTZ&-6RdC0xSfBShGxNtJ0xwv)OuSP9gV(@SS^Of zGiqan5){*SPIjbIRmR1^@z8x@Y^!Z|AYU)!OgmYEAz>u9GT$tT6_Or7AN1Y*FR3-| z8v-RGjYt-FX=&dUsZ;rqbaUl3Q6m@H*7VM__R*Jk<%%5Z7+KEEdR~EH_`2$FS?Ewf zaKSbV3{tIx6HD6#Ae&N&{9adC2YFOJBX2r+1zAvTX5H~NfJACK;Ap|Byu%B z`MMq+zfJSLtoAe`AyBIIa4^~vT)*r7=no+OIew9%;VOIdt#5A1whF$M9L<0iM?#`T zAJ52YZ~(FRdl`=14*^wlbnTaLzpm4r-IVXOM3U*syb=Zm@ulU;qgeux3JW^wRmI;` z=f^ytY6=Sn8t`nHzN21;IUzQH0U^=`$6t~_+!q8&S@rV*R>ZU~sZ{%Xm`L$+ndmb~ zR5p{Y-}#p4V4A7PlNnstHS~i3$BIiI>P7gZBq;2&3p8+2vQg=Vtuyubs|_df%$JX0 zFcC{n6+0a7zjZ%#XT5bkKaMeB#v^e~+ZMAL3x}YS3gq)i@Mz*ir=rF3U=Nbv0mXhu zGlSGdudc!X9LBUj3Ioc+a0H8BqB*vpgglSK3jkch{B2rx#R~$BYq?sYey z#p~*cDlkO+iM>j|Ga?nCk5}wtuzet+jlS+88xD^o&bx7Ss3Un0b6S$5c2|Tx5NGZVK+M_4-i>YrMb4`I%uzjrcx^3aymFJL|pvRxkBitRkD!{7^oKG(X<{ z`(k7Gwp7f!8IVU^HYrYGK5bS48Bv`vw*Ad!5>(K;hcj;e$k3G*5<27IL@W| z)CI`unONAqSqd5d^ihsPiE!5uTFHlJ8;EMMN{PLk!4qlUqFk8b9cZppgOMej%Oech zh9Sy>cb-E^ka0mBQvbN1Jd4PjE!MMHQJOTfTqn*l9hXvy>yJe&oyZp#S4?HZ=}c>U z943G4?LD4VSVD7}Rn3YL@hC14Qz;X;&OE-Hn<1H#qdt3Ak6sq^y1&r8k!H_jZ&NR zbX$aCIOol;y$))Q=e-wS=W2r5sQu}^D;&^-b4UZn+?3&I--Uq>onoEGlNw^qn}s? z&2Yq@p6z?D0EqbGMB%t(uAc{)5eShyN;wF)HCv7psV+U)tVAzZV5a* ze$qP-S{6^QRV<^We@fl7(-jF)K%y42Lc+Gj(6#p1kW@2p@5>#1J>1w}Kp^Kjh-Z52bsFlLGZ zAB4ihu2qGuYxynz6T5LUn&$Himy+nWT6x`BMQPia4LeA+c+9QMsF|o zA9BFpQ&tf4ra6s4)k}mcpJ}RZ%2Q2Jn8U$JOp;Lrm^;XhSu$wj|7E$r`hUz2CmvN+ zU07uR-nJ6?;Dtth(PhuDg!y6G-lDIY_vZrv|8lDn8dNk`n<<|_Oszo7R`W=+#z2nS zhzR%Xps^%TQ94`fAK6J7Nvu05uM2pe&Zv^rF0eE^gz9jyFl%c!%Vb%d!g(qg!0IDN z#bS#L1bss%oms`yf0!c&Qe$N92ss5FfKyU_BF%nYu=L|sayh;ESzOuJJhVquq~)#PL=kTx{Gp2Hnit2s=PTK%N6e>CWM-w~TFWMGgN1_A;JR6^o)G3r*0*(e39 zjG$7w4qdgz);ATU%Z|-Es>`35$R~bz&D-5~WK*_)HH2a_Ek@g2{l}}`W@7aY4Xh=U zZ^oFu7nnRa_XO6>fLhqT{u@-MKlte)NUgiX!${_4h0bMfcwtWhW z!um@xMyZ{H1RB@;T_TdyF6>KdiuQ+1iio=(hVd*n1+W68dRI5LRerm-v0Di`dN^8- z@|#;*;ny3nqAa7-UnLoBkt0)J z$qH!#xH6X6W*LbJHSmjxMoDCV(i5&$(!-H`Fa6&6MbdbxV*YeRqUG>F2X#@yp3~MC zmkMF`*(EL`&Fq_uj=x-NfxsSaLJ$@M2obLwer)VH9Gju*I4#C4S0zYP_5SH5vkw~h zkwKJAyKZjdgI*1`W%bfYxXukV@U-mHCz84L(%`kq=IysL$4~3aW5txlkD40#H_(uX zk;H|AAuUyKU<@Vtdp7f8{kX%$Y~Qh#$?W~M0Wol@%+BIU?h!Kn=tDgyW%I)}};fAM~1d^{_5+1@iPbub+Oc~7?K=N>(9@+6;^&`J{5IapL z1SE%tBuGYLpPL*;9|55*WaF*zUO3~o)t{EVtQH?{rGN6L z4g_H|M*`91c(jlLTGvxkIS`k_D*%e7`s((L1ac6(N_y2kO$s@Jcl{PQym#9~fE=dwzT$Y?rCj*o%6Rj;RHF3Ze7bHe z)b?FEJ7tNjq8ne7Wd!eYpYl?#dIZXqgt8_iyb)t`6E@zr%O2To+?7^0Y)D{^Y!e8( zFCG%A9KGgxFZ~PFn8(MDh`kt&NTm%W2G5%!G7UdH9B1l5?%i*?oh$flN5?8si~v?) zE@g*NbItO?OSUjqRP|FEyF+FAmQ!W(=-_Ybyro<-HNU7cG?f~PR#D+fmr3Z1=|re( zi=?DHL@K=C-jqQj%D3q8EZs~;0{?LuUE%sWjo{pW5Rb3n_|0M{tcDGf)ddarVO;NV zW%;^MWtpokcU42-_na8yIIiMOoh@48QY)d3&ss@r2M@7KI|)a5uWi&LO3We^xwk1| zT61mi(rQMmsSf-sC8cTo;4A@A5H=;d5k)VDL;^$zhB`>O#K~k{g=C!-f|CxFb$v3$ zROjl$SA;w&*&i2)7Y$<$@P5}*Y6q_VO$1|RW3+E=JvK?d>hcMn7*f45iYdeP1M09` zxuYsv{Me1Hc>8qp$8C(fy)?ThgwHrR1zPG#X7X0;Ziyk{PKe7E8&i|qP%*bp6HTkQ z8Vw5?!sYRJjm<0e-M9OncvFLds3;Q_iy{bn<4QsGQS~tE`u6>OA26u zss8u2Tc!|krq3ot0ZAVO^ky~YF{PI|d(?f@vG-4JmxtQo=!aSRlnZpXJ&4oOe+$_F|UTbK$ZJ^DOSB1>qr<5P#BFRaJUU^f2YN+eYK0v7L$4 zSdDEqc4J$OZGO}HdB2}9``UBvQ|nkPzb2HsWo;KLt(!0zebehg*l@K)yrp~bQpoK$ zk_tEW96*9@p?IQzUnXbjTzdEg$dFR_vq@C1|<$JwpVvkl?e&)ngneuo_3ECsGdf;R1)K*s~mzz@y zTpUI~9lT_a(37VTBS{n|1$0zdj-;A!QG|X6Wx2J-W8~4@=ZV0skQ?jIi;e@e3Pw0` zt~?d2w2y1yKY_I4pc$wWpcOtwh=GI&ZEj7vHh?aaoNVxr{4x^#y}4WG#bSop*BTX; zQ^JMJs^7a#pU*5hUsHwzW+zxh{~QOVWbJ0vZ+iQ;OwpuUSsN<7KU(pIJaeuo?8Jo9 zn}i4j=s%hv&BkzH;ID;F$dZh?Lol6QvyuO;Yv$AI0uaS|QI zJ^YG4Se{{YuWQm*uE-S?a~te75Qcb);cXhWTnzyg3vm}AQN6Hla&9r=KyVt#6&)c7 zYWIdt4tY$>F9$Lr(=~Y%jrHiX0^Pf*-k}kcoHZ6Yc=qA*wWa=Rdpl!+QDQ_d^>xvh zLP>ADTXd;$_9l>eOl@phwprGt<5H6)M(r&ho*esaaoxd9qM?&hh7}_qbuM*4Sj~ z=SsFCZMf}|!?QE9)wEF^h_?n=Ag|J%)eEyjSw4t>l6F|mq*5Uxn;bcM zrP|-zDAhU>8_xCrz&`g;{#qn8BF|oyuG|gG4E>K_K!l94Ov7R??-u|mTfrb{KR&BcpV=a&!i?#Zt96YbFmNy1E)E@DxLG=kh;62|Hxv6J|*0&Mqfpv)u}b)%b9| zwQ6eO&qmbB?g9zjxtf7h`=t%{amk3)g}=qJX;Bc!JX$eFl%TmvnwTp+p1jx(GR@Q9 zo?6;@b_6Q!pe12RP;mwRMjwbT1Skn6mirv>Q{hk6a}Q~&pvc}{ZL%HUlud1{P<-}r z{`?q&(68L+BQx_bkc@*}$+mLQjS1jH@pS2Ko^dzh`-Tu^HFLe@*nJ&lR3%ksF5C3d z$abc~SdKNRn^M-Akx@?d;lh1!wNDI&gy7-rF^bA4L#lwCE&2F^F>R+`AJzH7BlA>b zcCt@ni5&=>*4_dF@9!N@#rbN>mkX&U3wz-l_?$_G$Fp?lF5jSRHW7bicymI_t_wj= z`W&{j=uIsV~+$yO&$_|JwhpYa<-y?4IJ1xhrq8#YkIuO-7DNa8mIiKr}3 z$%nYNO(m(o4KiCdUscrltjzOk3maR#7Cs=bdaC`7^E%kf=i5Ncypl9(M_#GCloWPT zLhpjL(O=cmoB4|&DZ{0zT6mrWM|EehJD(1dbwi1~R{${K1JTr?urB#uSH#^vtyNqi z?*|G|ORP6!1y*d(Lg_?Mj}E=BsK`{td3zQzwTt_tmUxF534gu#_QfpxT!#P*@0P3) zQZ2C5=-$+)f#?Ux59&iK+j_x=Qk&@ zlm)pMT`Z&2?uaWi2$mvBhOC`7ldazi)sf#SZ9Pk7c4DANIM$%q8CE(gB{aFNqXu&~-%)k3;tySiv^} zg?QfJ=y~Ai1bFfosq$gQmTU)F8n-M7{eeJR6^6NDsFTe&rRlO)e(e+hK)bSXoW(XV zBvy*DIRD!Wx@+i2E;^#elhq7#zNUz@8IwV`T?WrawEx5A4?^qjAlAN_Z<4+*)SgWwe65X*AcEEvdZYabnDW3P0L|jC{4HA{kW;BF8 zJAht(tuK%qf3jIcFvK&jH}07Da4VqIJLrV&=FJ!MZe7uV&r*%F!q0w_F)hPUWp1AR zCTmI`+a_XkcF+g24+E|*!eNJ@_O@qUz>|=yJDrNhYGD^AwaLE0B(K`}GB#d*bBk0L zH_|9lM8avSJ-Gz=Av^!)B7Vj@kq~^v5eSt5i}R z8C-!e4C=EQ=B|Q@W>QjDjAn?(N3E~WwO_OYczuaSnwc)%Gss+rW0=F0>7y}y&yNQL z_K=mP@OMuaGI94arR@H9n!HGw*8g{$_OeZsA_Rs+jtHi1ZcRjqSz`vGI|)e)Y?owf zhmD~O$|u!RiJKg&vxZimIW!)!Tqjnc;_f8YnW@I)#^Kg0aR-^|r5bD;p|6#Lqbtfm zt{q;@l%l7TUW=9&44`h3D=m1l>wk&x%g(fxj44fQ72D9P7A@1EyKPM?vxE^ZjM>I{ zJ(P*n24)#W<}0C7ue*2hC*kKof@CsE2XZ4)$1c^65buQMIHxMSQTYlOQt`6Y^*xk0 zY6r6+#2;Hb1^b+qlr>3OgPy08oCD*GUFo0OjiN!I>_fxSu^fX9*-chcTc+sr@4xBY zKLye$Oe3n_!8LMIinOW&H4P2w>qU$I>AmOn9LUU;mRg2ntnhcr2>W5d3QES3xKJqu z(Sr+QT^^}Td~JN|5~sXKnvUoC4@%vQCxQ2UG~~RwP8czT%a)!BTu1tOtg?oqmC*sT zWXN|b;#TAXq#B|R*dmWonbV@SW3DmOdZOJh$ul)ouN(8s#~R(g3Kj0Sb`@k&y+3md zBT0+Iq!D>|p~$?vvtWF4z`=bBf)}93)a*p$I4G#+hS$Jng+wryxV%Y>ar20oPt~(P zxdeAD3BLW>cN&uS#hsJj_d82sqVAj0BRY1_&uzp9E0cVwDrHXUE*MgZDL($5GNgrL z3pKRftT}4Q024Y2rznyM7UY{Nk#jIfF!68Fi)MhOsE+PTPh3vVZ;m>`0IPgq) zVf;9*XP9}#zNtcpC%DLv`(1T+J)BW)Gn;~llF+|$x$(Q{VTB*J)G&HhiwvDAi$_Lr zh#`tO_kTl#5ma!8PJvy#7HI`mkI=V79qz-{z502r3k|gR;bVgu&&0VQ?T)gX*{+SQ z{_sE~@^-KZ(U`5+8##$FGNTe~G=i7e2Jv zvSlXDlQ-yR)2Kt5HRq(^NV$xOZ$_{wu8JH8Qr}_2wn6T5r|kHuVs_FW(;eIIRekvj zjndz#Hto|Km}loBXI2c`B?ybz8!ug_+F>=N6(T_cKvv3Z>~LP!5Ew0sGiu0_*N18?^wxyfpjM<|O6-I*_EX6`hp zJQr$u!;r|61&rj7leKknag^X!)5?GO;VkUDVM(shk+$ib$OessXNG{wAxC%X)a%x6 z1foIh4(!DNdOr91cav2W=N>mlxi7ZTDC5>pQz{lUqRnw__lMeBEBi;b(Z8A(IY!QT zIF<2tY9@SxGtuG)c3!{O?%s?uVBpferG3k+By-=e=fq(>xXs1hDklV^kDJ>Q)zp&t z^>tnFi{Z}J5xsyJq&F(q;_y`ApKtC2{Uz*0SorWPqt;YK5%ou({k(zPmS>J@QUV3f zJAR*2F~=tBt-gc8yt*Sph5o++Y$71^ztkt7PPgt{M1Wevn)S(K1&LF#)_+qZDb(No zVi|fGsH_VxVmc2BoX4;xTV2v6H?Xplh*l|Wx{)+Dq$E?!37r^Kq#sD5xu_?LOmM%c z9r|-CM)Eescd1HT9J)3Gj4NEzP26YSHnYQLnry1IJdxr=S~#PB7syNm>Q~s3#nRxU zMI1}bHKH}!>&7|2g!^Wt0DIG=J^x2A%JYK6vqskl3n#e9q$wU! zz>&3uFU)^+ziu!gPvPDgJb40JZB2#aL zZ3nxyk`f6$r;oPX{n6p*mMuS7BPu5uk}04*G#3aN+~9js#JuP=d~TfWzwB^`UHDZS zO%^^I|HxY|_oJEXe1CLzf>q&`985aRijmW$q*5^=wjqsUEM~(k6%~83S7_4I?JJ_d z2pFUmA+JB+TmO{l@~^O}-81czoZLF5SG$Z~+6^miqXPGAX-+nG=6~=7UMJ>cN{kPt zx=SxDk<}PyQ_YX3X>lT?AX4KS6~m2*QCZBPyq2&JH#-oVkbZk)Z*BPJ{$-MZ7rX88 zdw-$TJnJxE(X&)-uklb<$#m_fKSL{tnU!q3IxfB02`8lb9)9QU&PXdaIsY$c=|;wy zh}$>~HX9An8lOwSV(P9!il9jS#Oix2QF_1F^(K`aDT~u$9-MJkSk@6{UX~*(sY4l- z6AiP2mL69%mp&eOHl_JX2ve}BNh>XO=@(YTf(ZFP6cGsLRJb&CJLi~WoR6Ap1?Mi z=Wuy%AeYdKE7#~m&Cs&@HZHHKvQq9MK0UoqA{H|^9N~8n7OS8hnxZbc1d z6tbYFp@zmyB`H3ytA5IZ{ns~#`sAxMJZtsVag6Q6F7PwkEu0IiIr4TPF ziS07m|5|c|Jz|Z1=C}oX`|$yV74>@g8ar|lWZ7(fY~~A(MTb(;^%BK(@q~7_m6P51 za;l!si&cr?R<1+ht|Deyl7S#|!3*LTUAT>qBT}RL8D2kPvT3#qKLk0agSQkkx8 z^Y_`<41g*lX&yY^7yoU{*v|onw}>F(D4UF8=S67@0)_|AYdz!azlLJmFV%3yrV^}@ zH07}zDD)gGb(s#5_9;BsQjc~mv+%toYoDNSs!+2J1Ve&FdIBj}Om$&-Q?NUD8fy16 z@r~Pj0|UryjG(mh^z;(b#m)2@b26&@kx*}b_EX5Yk|adLNGz=X!O8;yvp5|w_f6z+ zL3k06u&i(%_EvK=^dx{lWh63IFA$PqZ2wI0B0yh>N$FN=1FF?IB}NFDq_^kzEch~_ z_*?kn7qDZa26STVfx7yO*Mr-@c!jkk0sp6~9Vj$pLA`&>h=(6QQFa6Wji!;W-MOj; zn~uXRemFg^qw)9tTQ4!pyrPZlbDAT)kTHDEsenQ`*>3Zeb;j=WDN|hdoeb1 z8k_YJta#?Vro$XA`;&m~LL1!iw5fo#Cw@VETO7Mw91L0=b30Gxnk|?$vY|CW@sr3K z){h_2pmI;2TT1!@>wQ@VG~*Ot+~Bgm=?Wlj*xi9JSE^oy>M*=R)c25)F*Us&S~QZW z9uILCYaKxSGbFSwV?J~kS&N+9*F(}wfOR9-(75oJOG^V6^!uhTE{Dv(?^yg(WH7NM z;!B%2zUQvH)QE39_9Fgi`&0e$2-iG zBhP8N=^4Ew-VxQ%?w296$gY7K(%1zenvrrFu*!{x)fw)?;WYgnFI4CI6Lpgt*kMqK zeqXt4U79`S@09L!{tT_nZd-A%8%1y8>#I^R%#*3{4LxB~ykIN9VCo9ElRRS{xR7=` zyd_BH`1(xW$2M7MSf*EY^@8i5rvGfw_4d^CoQc~Pbr*{FO#ZK)87+4QT-;dGR7CR{ zVtJs$YHKWZMC*=qo#gtRNGHfq49atU%EjRD^Y9f-thh*JEZ_D5a<+vlVvBiz* zT48O~J>ST87-Pv;*r5AO8LNpYOdeEoj~KUZVrJ%JmTY)(ZRA8qwu$ST6n+{FLL?d<;b=>p9wHaQ%jW#`J@ok3gMAW^W?_2Z%QO%S z89!gj0*va2SYxr&X33HXj?KTmoIQW9H<=RaJWs>zGEF}Jur>vs<$Re5B7l53*RR63 zWZSCN76_Y%$RdnFcp&`D4+8^RB8=w1)T={WPU4lOPKtyanh!~at*Zwy`PW?1siM99 zNFNXe)?bjYqqRgx*J_-`LMMi`0t49!46Ek_a0L4Vj(@uRF-2uTguBcl`XZ|xGQVK1 zFISz-p}3YOP{i(r*340)Dki33s@T2=1;zTa5-m%ATE2Q#5T;hZ{X>T}EPL2#4aW+> zVUYZ_n=^*<<8pR?#z{35gbhyE2R}KuS?CMh-G+ET)6@rP>YjE9BX9iPqdboGV%GAJ zrMu40%L($e47{(suBU^0l5Ts{Uc z%;u-S+&0*Pf{&RD)KzD+J$m2ldY2SQTxaC_{JHwCYsz($7d)6=_XCD!)A$Im)98Rv2O@p(56v$86x-8 zXmLC)ew@aL8Va~4UM%1F3Zlrb_7^gGWi!bp$?{wYRm$m}WQr~-G`a^8#t1stQCLH? zI!({ULBM^F5bMfa#!aEmO-3GU#^Ze*S zo1ekBr8!gtpSK>#zb>>Qp@`AG6K!C01)wv-o6=g4r2ER<4{PnbpI%VIHsGNp)>^YO z>~C$dZW0QTd`I0C;qiTWNXg=8kKV}vPYFFn2dB(+j(YhhaGRCilB!w%Cgxya2Z2mi zd=eQqJg-1JX*h)j>BCwxm2~=h!%oOZH_Z;3d3%HycM-_wS5l>0uQ5g9v4`vDoYZ<0 zbc@@4JX5&3s`LGizDLw&*CRpT*|t=DbIZ5(exwL6eFRYg+4+Tqs1#ZaqOW(FCR1mM z;v_Qz8B>M7wqcLrcrh}XJhR`n>mHj@WR$!eW`A?84DbgA#28l$keIr}*kci^YS$1y z*OPBcY>CtFOEzuMX_Hv3wL=CA&s0>(f!3WO`hr9N)Kn<>N=lFr!uh0g+?X+gh@6{b z4=a}Ii-z_O_ZnnmQ)SCW&||L|7IKPK5=;!iPQ*p*mb$AE846}eqL_gVudyH->KDa$ zyJijKu1M{wu6nDZrT9ztDGzLLRH|i;#E0c}SWQ_@hmsIu8u88>(chDhq)_0Yt2|6h z0fEBMm|Y=$A<+2EqeuKUsoHJZUwU`$VUtYnR?QX6hD{e=6$G0Lp~X%8p6oY*Br(_A zY$ZjII~@?IgCU0gAl$9C<=pkU4KAY8FXgi>e}`5cKNI5vz-Up#*7<#-&>DIZB|1$7 zGC(@$NpWiVqBcY-z2t}J)nDe2q&-m2hAVR4(!urR_~(6(??4Fb>l677;!@6b$_Q|- zJ=NHZG$rRTvqi3m>kZ-|CKN-aRg2_B4~15rR4<+054(gd)f7-3^cU`chdALvy-~|*G42~8aE!AVS8C{642+JpEe$p@~(%r`I z-c3y*BhOnsdByR~UjDYIv&3m%T?xQ6-4qKOtoQB>9 zjiP`Hx`>3q1~IhJhf?OW&Yuzr!|{+4K?lq3;DXWV^J3@}39wqi#;A5YBceOUu;2)U z37?C3dbmshuD}opu`g2J(jfz58&xv7K5VCP$p1WX2+zy3}273 z*9YTC0Jct^J0v=aey^kwer)!>h)hn%u(-)@O=$acps7br@P1306BoGuJf3Y1rpR|} zrmUotr&mYlt?QC@a4cM$wJU6)jaMD_pH{4f)QD8T5sz5Q1_MC7U0>F(2rPKpc?MQ) zR&$AT_M8g+t>wM!)&v0=$ZGa}->{2(d33q1F#MG0y1i}z0VM|DHkRfzfXzBOl>qg0 z&Z)CasG*uAtE@5?2xvr`d0T-qntv;~GgE2@iT3FdvrcAw*a|~-(Ul|#)VelbzmZE< zx8y$JYQC=`B^d}@cPvdEl}{ZAbxn0{ntr@4|B6nfq2+v&qSF$#i4ZNa*6BLnVxMia zi7=YBqs*Az@weLz-{0B&CPJd|E5z9DOMRFzfo`QN>>6Vhj1-wkI6D~;%%zi8B1V#BC?Sh#>@b^Q)u)4qMm~8VU@pM8wN3&y~qxL4F;VF(@LF;S2{MyHzuD&dD->09veMWI90| zMoi0VjbE{u`)T4#X1KqO_e4yaf5En?O%05Qpwh&D=s1QJyuTLi+RcLAMyzSmU4^Ab z>39IX=8PLL(n|(egfx)DKvf_yq^XzS|8k#b_nw?hf`vGO(e2KrJ3-b zS4qLueCRHu=X+L=FtVQVkX)GYBoN`=K-GD~(Xzz<*bFL?7?s!*KdLmEnGL0>kG*ai zr>IRa=MOSo+d#@tI-$Z3Mk0g`PKp z@uUwdpK?A0c=Sj82QihhQv+y5ANzeFP^#;Qm%XDQv2BbDSFbJ&wL`^mJK7fAbB7yz z2Sp2Nj9F}P;>Tf@W~JA(7iRqx`TO}$PXo8p_@VBp#I_$KXDL0W;+yw9l0~bOaMTDO z3-5LP>2&*=NVg^CYSjgj3%o8u$kTSj_Hbm}?<1F|+K z_Yw0m`E0}@O8jy^f!L`-ogwn$s z4;81qtH{}26%2OuyX?P%C-4z>a}I@Dvbw46jnJU-XTWk z><-2vm$0uVqk-jI07_Jd@&*-`_Vb9EQxo?S(&?6)Skmbi6qpS5+a-vJZWc{F=a26v zO)K&5il0}($RQBW4=n=8p0S9KBXBpB4S#;uJYG1vz6h^wf5Ew)QxvcmPh0T(EAa+0 z3DqOI>TATV{PB1ZhWoHklWt>jf%07HwNWchN84V@DNYB1sAhXsPu32RfzJnRTuqJA zn)o*>8bgF3;SO#3{;g{5&6gv%>2qhG+D4W|+c<0s9bqUZ7ra9C#TKjYLF}!hRt_O($ zdJ1mk;GB|X!XXDZ6prq1?+X(^2A9Nsds&kyFz%~@o0Xb8?{<@nwVaaH^2=>2O_HGpIf63*DxEhOn}Fy*A_d^^Mo#EcNa zx0S+hSiy-PNHX@Y0P?E}VEFT&GWOjlcinwwwV0aabXX?ZKRCAy)M~t-(GOmpLZj6G ziUN6mj4aRi;WrY2B}2+dPcZv*{)JU3>RW!n?P5q@E5r_ysMvw#5IF}*9vNRc zgbEz#qgHpFr?JC*6I8Nqw&PaA+7N>7dKmk@0DHdLcM>$`V(kyHA}jQc>4%z2Ky=L9LCD}YPgFu9uykEv4buT+ zE^s`sIH5`+p+%tC!eN*ju6CR9Ea0U7UC)rR+sZ&ii}*}VULIpJrzb$0?h1ETH(J5#ezd zKwN}7s!hcc4}h>L@9h&eqMl)Fk)Csv2 z@~G&SO{sbOPdl8YVYh*v;xj_6kB4p9g$Pa)$)Y$&3RVp8O7&jFRh2L6s%Z65`-Ce@ zgs=_YA*U{t%N_&XQ7aq>Ns?rC#<-+TBzMANbmsm_h4SkKzyt;eV=kCY7GYRHB?)D| z&4i$@{j3=j)NsT~n`U13xXTv}pCR7TfdwBX6inv7n(hyyquV6vdA}9V5}%g15e{Jo zEa4<;tJq}qnU8yNef9&kVVpn@#0+Twb8hA8NPFy&RK0tlz&hH>($U{>Jh?94YU@m0 zz*yNQWs81y69Q!YYP0zzotD%ET=)X}tw;Z&HuioY!q$p5gGH!(L+!5y z^a%Iug-TD6aW&m9I^H|Fq{JvuAp@ih?5SB5m}cyjBv`<5q)(h8CJ#|2>DHK()1N+ zuK;ptoaLkoDJODn|KSWmYOUQImYaa_3IcK&!zraX&qHRv*TdE;cBKVee<%BJ-8X9H zlA^|rOVShfW*zMl<-dA8mzDAfX4FUT?Y5E>LD??mV*jPj5h>zH3vPGr-Ex8KBSHx@ zA1I!aAHRtjFqET6x?1eG)wa;kNYA>d(FF&`&d7*a&06*K2yFczye)F9I2DHjZwaal z3nu(`ym*0J!a}z?Ene&P{2Go7pM%5kEfJxNJ~Z;8zKygs?U@@cm+#+I*bOF(V1VMM zYnKz2**tC*2S~)1yVi$k5$CKNIDf3gYEHwlMX#|#OXgsyxntu)DwBR5_RH^Tsx8`9 zcNi#2OXrJrvx9>hgI1q+#ZX}eQ!kx?@b!5x=5oE&G>!E;LOvrpV*9BRfd_xz?h8j> zQF-52VGitLbWe(j2CKPjH|J=*fBNTPf^M-Wer$We|G&kZ%WM)EyGJ=Qqv1{czjkBb zQ~62B8sAA3Xv*M;7Feeyy@po3?u;1-f9ufma7!I=3oxEfEaiZKpk};7P{i~UZj+O5 zr<8xPW?$5txnU@v-(2E^s37}vC0dz;yUxgO@8H6LifGG&mtB9J$e7Seu0OL^>@03K zeZt)?)_~u3u9BO-j@0gWSg8^O7 zy;1uvdm^hK%CLffdTg2hI)dt@YMoNsBA><4SJkqyoO1qg=bthN=`eYnW&B0Qxd!;A0V&Gp*W#p!W?wv{U#lzG*zLTlt4W(T2VXf zK2KVFyS(A4@4U0oIoAsl%yDVtdbBY&(+h)Y$4;(goGy3Lu8uSJuZ4>@L`fnY$UYqj zAnrx>WqOXNUav|Nb)(raOKNg5`nk)~^Y?%+QSlr9~%px#bFqw2(1SQ~KSf-YsJ2Ii8NinJs+q!t5B&qRPX>{ID zxP~adlV$8FF!Y=;)RlVI)nw1{y6}2f`i{jqpzf;pp_N(h8l3J(v`v~F89+hO?zLXq z6$|Fv`DK>UJ8ABh=RGO$sEy0$*HIWm*T=5MWBm?-y{@<(HZt%8Akfo*g9nIL43hz) zzMw`e6{iy37bT8A@DnpVqLQsenX%JpB}f~>(Qk!*gn@~NS(GiGC=)4Z3pt+K<60W6 z+siu5Fh;N+DvORNc3G5&NLjL+b|K^BAM2uO4Y8vdMuL%0?9eRs)wR0nPl`(h5$a~0 zm|7uSLFV-7Y)iwC|8+Tn3O~^`aWHKR(C+>uf^1sD*6k2jBsq5gxSjLr z*UcA|!)I9+GMB`$gWjI5XJ*N5DU^r_7L!r$>t5ezImLGLe3Pe$1Uk!6mDKcfPcr2F zr2#7NM6$Z2&Gg|Z-cNA3F~}ts+M>~DKa@w@LByikD=pyUY$6}8dqeLO;TXV58peta zIa(Hp6b#l&2qPHdm&nf%ok*K0-zmP&C5XPIZ@3|xTYj39EYXhF!9owF@eNrH-B3sk zK@UTmv$Bdcz!vP{$JoDOd}v_uVr-*d0CV!^Vdudv7@p8FF`@)DsP2H`1l!i~j;$2` zWkb3{6**Zm$5Ni>>irtT5TOt~8@EL}m1RQ0 z>uE)_sf^q%*5bbu?eKtryH$|O?3t(&RQGLinc}AF%g;Z*@Ps3w<|ieERI8ZY=s3KM{rVThrB*=z3L&HxJf(K%=aLhTuRH`wI_(PVHAp zS?f(jF2j%rRIr6fd6OUjx!G!zw|O@8|YGl;F`D=2I41z0&i);_fY_@5k!2xNr?m^EN# zOCgJds>ImM0^M)+QZ*wYLUe1G9w%GwE2_Io+Tj1>5`uv%3fv=fyym>du%_9g{|y|_ zwd}m6B+5vzo0|I&lR{(O@bT`Mau=qc4?`fi%Q3Ff&8A8i`?$n=W>?3;1My<2sy2^1OBpPuHVoU z|D{hq_)RfmQQ6AnxFD{2jDBb_RMVFlxc_|}8%P937|sUcpB}*h&Ve!LwY6gV$e|lo zC!t~_D1f%}fX1kZk&Ffd=3Oyo*~Suwnvony0z}o)o)L$RJ%xJA`84}vgAVWJQKlP~ z)#TNbL$?n|GQiA#-^eaHDvYn33QSvALkP1%Nu{NSj^R~#!r!rrD;_9Rsq+2?&k%!v z?x!IJBPJ>`ix`ZGkl7o>z7U(&V3Sg9qtLntR4{as>uGx{ax%Bw&p25WOP7ieu% z(4ONH(GIT7{1x9&x2V^KDd#Kvd?iTyWqR(nBjWhgzEC=duYy>B;|2V8 zV)quu_RxxUn*EdlfzOhLVb4L6pIVt>VR!5c*U70=352eYGk85QPUu!)F)2zfOVGLt zI7oZtOMTJq-u#@;(otJuI;>!KfMs}$_3ox83;j!c(Q)|-@HK%CME#Lq79`X&@qX)x zJ|K)8`$|e=Dq?J%FbPmDZoMG{$i0z)s%0WN@NbVU5uR|_zrrF$&8T2B*ex5o$W$grYS!Hv*W^$J>#-N7a{Q3L>$i_u8K*TW- zY%?|je`f34XD2UY6L~VT7usJ zdi3O(B^YIi8ju>Lkjx(u%345=s%3>iMJ1J{N+Tg%Ca)prK^0C_kY&_&+L+m`|p7FWC ztFQt``sz>w{wgvvdj0(|Cnw`9qG7dFtv?L%z0JQD!4hS^-0b3gy=!1x!M6N4I%4j% zm^KS9wu~(vlXZ>fq6BwsGK{B2{sG6TJO;a)Rup#PGSzVC^#dtdP^U`6`1;4?X`v$j z!Y_(X7gv}ONzb-4BdsFzT;73%9ERiiF*V_&jrL#d)7q_O@^U8SCUa003+YDSD`e6* zWp_#!;%p5>^;pnb-PsS#DbQP8J~ zA&!OX(h&(ZY^O;6MNRXioL%UFguVne)wR z+n7F&e0O{kS2%VkW$hXuO^df##yQJN#VaD-tsz@+lV_@EucQs430xKxBtFtmPMOI3 zCWN$a78)VEx)G|IRNmb4{bz-fi(#>=hhWL7>YL0-3WSYwE|(hNQYwT?RAeR_Zhu#rAk%J{)Kw!s{po`8g$-7;y*TC811Nb($AgytHwKHoc_<87K@IhETU6 z0B!0c8YJNss89uYi4|RZUpNk%VFDbLew`06d%5L%4ZvF6{0-zr6?~l`+ctz8eVes31<|$282AA@cW|ctp>&p6L%E zz6WTqG8WdNDVqsi`3TJ86j5Fu&S5_Doc0&6r7rzh)UWjV5BVSQS)UAaI<0xI%8p*~ zJSwP(z$;XQm5TycG*=7td9&Y@>3S3`gJkWlld&<&B)fbt^Z8^Ea8h4uf#51UAR8&F zK<68P!ZWOW_DVm&YQ!{hn5=?75RsDbsYJAA_x!wF?aVf7pGHwe316e(u%0NW?I@KF zsl)8MMuuGqs6NDaOI_vkx9>o^S@QUH5lyL;>p9;`9#K1Vml#jPhc2wCAN18k7`!ge z*(5)3kT=g~WTH~ajJTo?U)PB?QM&mUry&64t0N*R+&a_s_{R+~g-0`GypJpYz9~e? zjDBrdEbHg%gTqL;PIpcAMRhInNO(y*$R}Mm#D{701Vt_~1$Dnad69E#c4xpxb&a^O zPg$NL(6yYHpv?z`Ew;|If?@-B^sQ8gqk^8yrQb}qxW3P9W?eX{h zc}gPw`FvDobGr%@x04Rrf;CiZtisUA06l&IcbkaJiMN!(WtyWkGWV6?&tBh?F|cn= zi1oh$zd>n7iNnd)it^DwP3jY$nzX6T(5zcns<&N;ln81``Tg9J)d+0|wS{{D5d?uF zks~s1B!4pO4DNDoQ|LXi3)fdW6m##?w<>2!<+Dww$ko-=X9RN$XQ*J5Du3xZbj9*N zO_TCy*CaL#>|PGGaZa-O>c96sscMx+PyMei_T^t0(VN}_@Xmwy{?s1$P^Rru^vee} zk63;hihLG2GUn%P)6+248r`S9XV$Hw?`kyV1_(B9u{RWJ>9OudtNXf)xf9PuG-FV1uYSukWfI9VmqafZZ+4fHm`%YeRZcYnGn zJJ(!=wYENn_`*hzd{}uNmz%sZZ2NY|58t3XSqo2t%-Rv+-#1Dk%`U%*$f*C)X;XPu zZ8rzl<>5bW0n|ey)Cv%jbS8X_hxaA<;8?&>XlO`p5}2_(zf!(%c4W2DOgfYWp}`Xz z;XFCoKabQ~l|{mq?@B49BlKmC5>;fN*(U@j4;4hWW7|_ki&#AaS-z&M))rRwNT?e4 z*Tdo?%6a6Y0(Od6@&D}K01(q>vf;Ujmd?JwV+I{LOEZ*8v+zPHBT_*JMp%1ywr#+Q zH|{yvvJFOxAwgz~NLw-JW1KmOSCE`a>L=5>cIZTMCv^EJ3efk^*e%$2T3QlT{qSTP z{QK7#QilqhIAY|>B0#YU*iyYr_jx+Wg%-y7;WL!PKp2NTb&pz?Jl24vmK)k zp7@uk39JdK3ARbTa5P+#LuyEhL+ViKLF#LpkJxmGq(59Hma|X~{oBw{i$o9i%lV8W zQ!?K64AKj|YR8Nt?%lQWBf$9Ji!a8#w!~JbK+Frv|CKf`JcWPuAopivzxF^#=HY;; z$mE60#{;wj6|Ydi?M8 zpM~d$rz<=hLEhb$^^Dy*_9#FHajHMh!rsMOJ#_I4q(s!DG%H<7EF_9_(;}oCR%eh^ zjy&5YPtttM@(7$rj5v}+)qtzG=O4^p)$003lCtBK1qjKejnyhOM#tZddsoheVzYPxc&vf&^@8PJ(dhRTUV(YN?t(>wXNc`7Agd0X*?g=T9{- zhwoMUwl7s!NF(?F7O*W8@ns0E4q7f*iX+~V-_n4|GUY9(08X7Uh9|+d#UWT><+u5> zmHlh^_$&<)4k|qm0ZlzMtEz!6Ufs;Qbjm?nT^~ufB9V<4j8Y2#HyiYZs-Z=knK1D+ zHK9Pd2`7ljS2(Q1)|NH#?NLIAZ;^Va2_92t}zSZN?F&pc={Cmf-rBh(vQIZ3Odp5MtHl zoQFD!R27CAI=w*Y+gH{kw>ylWB6t|aAIZLATHFDl4EJbS8({P+V7tkf?C7cE zRfO@a&(EQFfweF|mEBl?_7YDJ9SL)MfVx;!x^l9<*Ne<^S{rA_Q2a?*uKxq~-?3%4 zfm1+^_wM&@5q!+sLczfL)X&!*)GSKB_js?8=d2I5dIfI+kw#MU z9CV}8T@DzjDEQ-yJ5^^A1t9`o(<|QpHKJREznBWNSy1-lbHKaU9i4q69wAAL31q`t zaQRd6G)?*b=Wf-F(l+<~)aRf)&blPf=wjU|iOX;kdD9D?w96ScD=RybdbRL^=(JAxdiW zu9LacC8{xO6W`=Nmsy&`pF{o1_dL&f)_&?>6wmiY^f+`qDv`bvMn>EfUrvl2MbWtM zZKQH&?QHl4bi-khelApB}qSy6RsX4dUlOSUIp1 zlbSkwnzIKK*})-`AcT`5FLTV~-su56toEaoJ1HS`dBZ1ieG;?M!kMn{pm0J>=}TEw zMTz)vcgB~=4e3-sT*G4(v=7f92EZ16fMdZrfhKU~)Nr=b@B+=jpxjhRos@nEZSWwU z?SV&*5MPT|THR$QVKS;!>L(kgR#S8*Trtq&o0;`a@}SfC!Ey{@Va9K=t)FRl0JL2* zTs3BmCx4*cl^gyef>SP@hS!*LbGoaqxIFK!ck4)zpWA#WuZ?oeoB%KY=_XIDDf-Q0 zPr5Sy(Y6Ter67*OLy3KIQRZMKA74p8ySmzLJS-zMfU~~j+AHJHE+Tg^*E_UQriI`e zfx>?(^vsXcFO~9{@mr?d{Nm23<%OeQP^i`H-M;GC*XBcOjI2eIi<{<^#>L#p%oo1i z?CE;P1VZ$;M$Uu}%Q}<%AK^wW*5UW=Yy)i0hz|s4h{rijw-?32Tim&He?&jio4->3 zZp-^N;i~qLXn;i97BopM(;rtM(M}YOIgbfntBq|@s1tpxj6Jv5EgDwYZDfz zxk~krDLRE3!l_T+CPIcI`%dzSl=!Cp9JuSj=Y)-JOVN1I!3uJM5071@xts9bPU7`t z469Lm4j8dzO$s$7q-bg2tjA|2aqicmXaFPJ@8YZAYE z&&thvZ&_l+!o>#quMzIM>Xrk9#TAr+Wd4>yWJRnd};BH-u1E)t;mIY+-jE!-0^X3o6`gQlhB1EuNaX^-E1G zm>Sjn#8Z)Dt*I2#3NkKSce<#b)hxSwWwmQX<-K~ubUEM8D8>~`J}Z5C@+n%-fzKqX z8*^*4C86Ri{+{+)$cn*&2mcLnUa>Px4|PqUAdcEFpD3(_C+#6gz@5*rodG*zcA}je zFZ-;W3LOo>x=mzTXn( z-~?Vd-vZz&V)IZ>h^BV8xZ8A=N-&N#l@e7D;p-YO_qNrNlqknFC`II*bV!+_Zn&-Bnf`$JV*pJg`)&kKcdb>*4UK7midUmOS6 z`Xd@ODyrovSy!?KH&yyQ?H}L1{&4k$OlEpg=u?fLOY04Z>Q%nd{dptHfX=p?P5QI7 zIQsdziYHI}4fjhhxcAZYMg0nyC;S)b{^7cMs5VhM8_P?kpj6l@*OhYP@ zaUC9VfthG?M&gb%%YsX=tPp*cm z8qnhb4b}1f*T-r^-^lO<*4@NSslE(wk8)^a^>ApU_4u6ZZI?CcV1y9YFQcvpZw}#( zkIiWZZ|bdP;?~EBf>WJ8{~yI&L^ZWEJ-1`&;TB*F$S zKfI8gGy!JZKm=Fi{l;?8e=BMDfksyJoB!nz(*i95&f9zq?-6QAWE8K?HLbTWhqNX|FPdL%9H|})psXa(OZUsDJBU6PqQDkSVmh z@62u%msEujaj8lHp#CEnuHn;`{=J4(mqU?TvlHd)&otzY3zFv2pXQ8=O7A*tZVh%= zwQVf92UTRAS3KKLw?sGZ5TNfGn7FdwQ5HHUnXhMJ3?A8IX9>QMt2Vwxoz*IHEK93y zdpwtOwyGzZk$P-$h%GZ1L7NzTrjxns^0crJR9;>iZD0sQ`GL+L8U@u-3OQ_f42jv| z>4eSiAbm?In^mPD8VX*8X`{7C8dtGOGKUWByW{UOrbv9tia3C~)BxOg1ss6}GjA9y zD{&Xdv75s%?|XPtii#en{PDDx9fz2k*-X@?CPj$`4YhG{kou4%O*o3j8rB z&IRVPxdyHjC(l{yy?wwMZDvo4Id(INNJ(`9a@w$6v>_hYXattHm?K#zAz_r{SPs}S zk3OrZ5j^YN%KfMAu?Noj}jJv|uFx_1m+Ujx6_wi_H2=}FQfL)UK* z$hEYpTF7iU9>@9ApBW+iTxMm5)tE1!bex)Q{k*}T-O@imW`9pcX)gdY7uZ&Rox(O& z4lu(UsOhTqrbFiZTi{Ev6(cb+oB-wcK;zUV!uaTP1^FR-;tw60-{PVRLVU{EdcTv8 zI2~j^h4km#&^Q}^k6qBooX+sP+NMvj`eKUf&-*x=?uds^MKedL9)#4KS04@USQd(C2eLSqr|f5p{)*WawAar#=JgV% zlIlBI`a?f?(-FBs`g@eYYwb@zq~dX)tZNn)(#svBmumrZ_4UD9RDBrQ$UrX@WS2Fi zhB3jaAC8F{XlvJR(}4-NWjeng$Thg(~7FMDE%9cf|}HPhL~fn{n&APKEw4^x};UQ}4FGtLO1}P!#A9VLZeht2VQG7Xvy;t79ofE=A{> z0$u%|<1L~ew0ZY(RZQY3O$FbR+PNoHv~y#1;rg2sM1YIls(0QMt@B?rtRub}&2INK zb2#63G^`?kvrJYzc|G%9?cq#SV4$c~hx|-n2i8oZhySpVsm784i;KloX7%<))w(5D zgmo+aSYW_BAAbyP&sgms`DL4j6|(Cgn&Lr*S#^_XA% z>ao!+z7*2>&A#uVbx~kZAU&hP7V@dE-nx$clRZq!Dpfhj1y;=;1+%X)u;+kHXtMCw z!K}#lDaUju@xGP?ZLKlenF!o6EBx5<36EL{wR{M_Sk)3T%kR$aY2W%6<;IQ;x{r$S1NseuL#;g_kCwK-eFhWO`)1{!;jj7=2~2nt^h9V_^g1%X=C_#iFn9dkEqu}#@Gd6>O( z4b^G8T2Yw+Krn9xLL@$VxY-p+5f?;I>?rT2>0Oq}T)jfJN)|nDML3n)E=vSG#CqXAqQu=U z9iQ-KML@?`f#)Tnp>-v=mQCW1e+f8?QdRUjHh# zMcImMu==T_9QNc$1D*kGB6~zr^-|-5zr#C|*JVojGdGeh-{1Une6Cr(Ru|H;@^`Np z#5Sf3pbf3U3)9BOpBSap`E-pAwvtZzjOFbfKEa0?|2)=*p*k=6 zmf>&kp9h&kmM{ z&-S}xjU;8O12M*Aw244SbP__*-TRRZ+Xg!bG_9j0`DzE(QFSj{>QgME)W=Hm=11$c zKB}r6YK;c@KXlTocjFZ6e!uySF=W*ggNV`4Kv4D3tt4yak$dFO5_vF3eM&Mx+e5h`Uk7$E(&M z)NkZ{_T@#VRmEC-<@2{p`=_If2$SMu+*W5+`_44Nw0q{0`njDgnq@C{RP0_IllJ{? zS$;G3jrknvD|-!FdctS_DKP)K|0OHR;oMSZDuz^yD(y}}zEDD53&HTz$S-F1mdd5v zw(5Eu$IDzW3+l$oDg(sP=yN9qn>5R_M@`|ZMh-0)_1-q~`xA2lZQBXF2_*iiWQl-5 z3a>*1R*I}T3Rpntw9?=?qGA@?l9Wd!TA^Y88dUnKqivlr(S(2qD7=uxB$_?^ZZIJcv1%3GYtVHY3gULX zQrM3BS_hDN>q`3jSpcouZ!p(OU&*4=E9#1ZBUzq22HZP-t*h`ADs+WwgS6@xlQlBl zmKML8|4(?MZM0!0ARo#zCEbJK?9`0(nK%+}ky59dQ5xQv*HZtgF`+k+ zbRz0Z9+r$O`_{)QlsRAUWW=~YY_Q3I8jsM-1_VPMr4?S(958m^5$`5K{BjL6RdQ&J zS)1fh8r>2Kktr6JEOtTWgVys25#h&-(|Fm;!g($XEb%RpMOY+RP7Azr@F5h@I z15ctF(LI6Nz-o?a^_v>{C?GmVg5_H7EvKIUqjv+a_ahxdGUUUcJt8dGAIb+&!j*pzgR~Oaa5{@ z%J4c^cEbknyVJf$Vo0?WKrJ;c{}@TH)xu!f7-%KZj7*$h{#S-&@1q9#Bt5C&Zpejd zOVse-PhBn$`)sfoGFBd_;0J4bO|8B+W2pW-cZhtFLb!}>yxd9vn8~j2j%CdrCKM9o zSS|e$xbW^B3jJh~ipI3_zy_|HVw_^x1^JEc88UxkY+wQArea<00)tOsYGTRM=yw}o zME}Mp0ZlgQ2)t3G6mj{b*C-tkER93AP8-p|k8`bP#z@q3lW$PU)Ys?1I^F;U$)S5;QTcc@IAMMu)CW24+i$oXin=iOu z%DhMmKliL3h#YD%>K(qy^Vjre`B4b}ZxLaNbTokWUO8aaDUMJvFZeT^dO!0r7J$+U zFKfx|g}h|Ep0*iz{H?fhesUOkgW`iCxz_d(pQ#eVeLb5Jh6FA!W&F3?m&ti)P#xW& zV{eRp$t#ySfwrz80Qk;StQ%&=6dJ@Uooi&g3bIvH0I8R&Rd8pCkvP>9T>m?M**>`$rD_QTu-t)*=2+2RU+)a_v zSs8u$Rp~?|D=hl7Z*61k?+!#DMt&ezq)2^~kOqvyGfu@BjUbmy)%FT!Po5wH?-aJsuo24T-zpRhqPDM`@gx+O(6JjAWj1^Tu@^$Lixt zL<9>&Q?_>)^=}nV*XZQ_A<)o3h=e92B`A{mjFS5K#h%=Xta)L2JX>lEKOv26J6=+I z@VQh#20@#7)$)Aa=^#Y7>-X+Crms%8X%{?y%kMu{9SqY;gImfOAOeTEsS}jyvp;}2 z6u!^TC#dRCt}K{hWBgE!@p%!O;|0k5CF%ZI#KJr{(`#}vb@n9kyi2kVb63`p1uDd1 zPXVH`YCdF8ZW?acqz zpo@+4#c0s*#UBPHr2tiRg`lK(oN?E*6ro10bk2n)Zv4)||R@=p~1 n2I3z$?4L>a|6WNs0oQWo%nOl^$9|GvVIG)@mU5+{RmlGUL3Z|4 literal 0 HcmV?d00001 diff --git a/doc/vuepress/src/.vuepress/public/assets/image/features.svg b/doc/vuepress/src/.vuepress/public/assets/image/features.svg new file mode 100644 index 00000000000..6d62739369b --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/features.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg b/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg new file mode 100644 index 00000000000..37fa923df33 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg b/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg new file mode 100644 index 00000000000..d5e64918546 --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/layout.svg b/doc/vuepress/src/.vuepress/public/assets/image/layout.svg new file mode 100644 index 00000000000..da754b58bed --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/layout.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg b/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg new file mode 100644 index 00000000000..72056c9cdbc --- /dev/null +++ b/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/sidebar.ts b/doc/vuepress/src/.vuepress/sidebar.ts new file mode 100644 index 00000000000..644874414eb --- /dev/null +++ b/doc/vuepress/src/.vuepress/sidebar.ts @@ -0,0 +1,9 @@ +import { sidebar } from "vuepress-theme-hope"; +import { load } from 'js-yaml' +import { readFileSync } from 'fs' + +const sidebarContent = load(readFileSync('sidebars.yml', 'utf-8')) + +export default sidebar({ + "/": sidebarContent, +}); diff --git a/doc/vuepress/src/.vuepress/styles/config.scss b/doc/vuepress/src/.vuepress/styles/config.scss new file mode 100644 index 00000000000..f91061d7ba2 --- /dev/null +++ b/doc/vuepress/src/.vuepress/styles/config.scss @@ -0,0 +1,3 @@ +// you can change config here +$colors: #c0392b, #d35400, #f39c12, #27ae60, #16a085, #2980b9, #8e44ad, #2c3e50, + #7f8c8d !default; diff --git a/doc/vuepress/src/.vuepress/styles/index.scss b/doc/vuepress/src/.vuepress/styles/index.scss new file mode 100644 index 00000000000..912abc5f90e --- /dev/null +++ b/doc/vuepress/src/.vuepress/styles/index.scss @@ -0,0 +1,44 @@ +// place your custom styles here +body { + color: #1f1e33; +} +// place your custom styles here +.custom-h3 { + padding: 0.5em !important; + margin: 0 !important; + color: #1f1e33; + background: #f4feff; + border-left: solid 5px #58a0ff; + + em { + font-size: 90%; + line-height: normal; + color: #2980b9; + } + + p { + margin: 0; + } +} + +.custom-h4 { + margin: 0.5em 0 !important; + padding: 0.3em !important; + border: none; + border-left-width: medium; + border-left-style: none; + border-left-color: currentcolor; + border-left: 3px solid #ccc; + background: #f0f0f0; + color: #1f1e33; + + p { + margin: 0; + } +} + +.small-bracket { + font-size: 90%; + opacity: 90%; +} + diff --git a/doc/vuepress/src/.vuepress/styles/palette.scss b/doc/vuepress/src/.vuepress/styles/palette.scss new file mode 100644 index 00000000000..01ecd52a13f --- /dev/null +++ b/doc/vuepress/src/.vuepress/styles/palette.scss @@ -0,0 +1,3 @@ +// you can change colors here +$theme-color: #096dd9; +$tip-bg-color: #F4FFF5; \ No newline at end of file diff --git a/doc/vuepress/src/.vuepress/theme.ts b/doc/vuepress/src/.vuepress/theme.ts new file mode 100644 index 00000000000..b8db7b12920 --- /dev/null +++ b/doc/vuepress/src/.vuepress/theme.ts @@ -0,0 +1,113 @@ +import { hopeTheme } from "vuepress-theme-hope"; +import { searchProPlugin } from "vuepress-plugin-search-pro"; + +import navbar from "./navbar.js" +import sidebar from "./sidebar.js" + +export default hopeTheme({ + + iconAssets: "fontawesome-with-brands", + + logo: "/assets/image/espnet_logo1.png", + + favicon: "/assets/image/espnet.png", + + repo: "espnet/espnet", + + docsDir: "src", + + // navbar + navbar, + + // sidebar + sidebar, + + footer: "Copyright © 2024 ESPnet Community. All rights reserved.", + + displayFooter: true, + + toc: false, + + plugins: { + + // All features are enabled for demo, only preserve features you need here + mdEnhance: { + align: true, + attrs: true, + codetabs: true, + component: true, + demo: true, + figure: true, + hint: true, + imgLazyload: true, + imgSize: true, + include: true, + mark: true, + plantuml: true, + spoiler: true, + stylize: [ + { + matcher: "Recommended", + replacer: ({ tag }) => { + if (tag === "em") + return { + tag: "Badge", + attrs: { type: "tip" }, + content: "Recommended", + }; + }, + }, + ], + sub: true, + sup: true, + tabs: true, + tasklist: true, + vPre: true, + + // install chart.js before enabling it + // chart: true, + + // insert component easily + + // install echarts before enabling it + // echarts: true, + + // install flowchart.ts before enabling it + // flowchart: true, + + // gfm requires mathjax-full to provide tex support + // gfm: true, + + // install katex before enabling it + // katex: true, + + // install mathjax-full before enabling it + // mathjax: true, + + // install mermaid before enabling it + // mermaid: true, + + // playground: { + // presets: ["ts", "vue"], + // }, + + // install reveal.js before enabling it + // revealJs: { + // plugins: ["highlight", "math", "search", "notes", "zoom"], + // }, + + // install @vue/repl before enabling it + // vuePlayground: true, + + // install sandpack-vue3 before enabling it + // sandpack: true, + }, + + searchPro: searchProPlugin({ + placeholder: "Search", + indexContent: false, + autoSuggestions: false, + }), + + }, +}); diff --git a/doc/vuepress/tsconfig.json b/doc/vuepress/tsconfig.json new file mode 100644 index 00000000000..e7496b008d1 --- /dev/null +++ b/doc/vuepress/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022" + }, + "include": [ + "src/.vuepress/**/*.ts", + "src/.vuepress/**/*.vue" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/setup.py b/setup.py index d5be9f60291..24046aa35ae 100644 --- a/setup.py +++ b/setup.py @@ -114,6 +114,8 @@ "recommonmark>=0.4.0", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", + "jupyter", + "sphinx-markdown-builder", ], } requirements["all"].extend(requirements["train"] + requirements["recipe"]) From 2128600b44680491ef2f45b1ac66b091a00fca99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Jul 2024 02:26:57 +0000 Subject: [PATCH 02/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ci/doc.sh | 6 +++--- doc/convert_custom_tags_to_html.py | 7 +++---- doc/index.md | 2 +- doc/members2rst.py | 5 ++--- doc/notebook2rst.sh | 2 +- doc/vuepress/create_menu.py | 7 +++---- .../icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg | 2 +- ...eech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg | 2 +- .../src/.vuepress/public/assets/image/advanced.svg | 2 +- doc/vuepress/src/.vuepress/public/assets/image/blog.svg | 2 +- doc/vuepress/src/.vuepress/public/assets/image/box.svg | 2 +- .../src/.vuepress/public/assets/image/features.svg | 2 +- .../src/.vuepress/public/assets/image/github-dark.svg | 2 +- .../src/.vuepress/public/assets/image/github-light.svg | 2 +- doc/vuepress/src/.vuepress/public/assets/image/layout.svg | 2 +- .../src/.vuepress/public/assets/image/markdown.svg | 2 +- doc/vuepress/src/.vuepress/styles/index.scss | 1 - doc/vuepress/src/.vuepress/styles/palette.scss | 2 +- 18 files changed, 24 insertions(+), 28 deletions(-) diff --git a/ci/doc.sh b/ci/doc.sh index b19cf049bc9..fbd6a51afa3 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -30,21 +30,21 @@ mkdir utils_py ./doc/argparse2rst.py \ --title utils_py \ --output_dir utils_py \ - ./utils/*.py + ./utils/*.py mv utils_py ./doc/_gen/tools mkdir espnet_bin ./doc/argparse2rst.py \ --title espnet_bin \ --output_dir espnet_bin \ - ./espnet/bin/*.py + ./espnet/bin/*.py mv espnet_bin ./doc/_gen/tools mkdir espnet2_bin ./doc/argparse2rst.py \ --title espnet2_bin \ --output_dir espnet2_bin \ - ./espnet2/bin/*.py + ./espnet2/bin/*.py mv espnet2_bin ./doc/_gen/tools build_and_convert "utils/*.sh" utils diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py index 175f6db0470..63f39de7762 100644 --- a/doc/convert_custom_tags_to_html.py +++ b/doc/convert_custom_tags_to_html.py @@ -163,7 +163,7 @@ def replace_tag(match): ) ): return f"<{tag_name}>" - + end_tag_pattern = re.compile(f'') end_tag_match = end_tag_pattern.search(content, match.end()) if not end_tag_match: @@ -189,7 +189,7 @@ def replace_tag(match): ) ): return f"'<{tag_name}>'" - + end_tag_pattern = re.compile(f'') end_tag_match = end_tag_pattern.search(content, match.end()) if not end_tag_match: @@ -224,7 +224,6 @@ def replace_tag(match): # if the tag is not in ALL_HTML_TAGS content = replace_string_tags(content) content = replace_custom_tags(content) - + with open(md, "w") as f: f.write(content) - diff --git a/doc/index.md b/doc/index.md index ac7416fb4e6..22ce69d9cad 100644 --- a/doc/index.md +++ b/doc/index.md @@ -194,4 +194,4 @@ footer: Apache License 2.0, Copyright © 2024-present ESPnet community journal={arXiv preprint arXiv:2309.13876}, year={2023} } -``` \ No newline at end of file +``` diff --git a/doc/members2rst.py b/doc/members2rst.py index 579f7b6a8b9..1e72a77788a 100644 --- a/doc/members2rst.py +++ b/doc/members2rst.py @@ -78,7 +78,7 @@ def gen_class_rst(class_name, writer): continue if not p.endswith(".py"): continue - + submodule_name = module_name.split(".")[1] os.makedirs(f"{gendir}/{args.root}/{submodule_name}", exist_ok=True) @@ -90,7 +90,7 @@ def gen_class_rst(class_name, writer): # 1.2 generate RST with open(f"{gendir}/{args.root}/{submodule_name}/{function_name}.rst", "w") as f_rst: gen_func_rst(f"{module_name}.{function_name}", f_rst) - + # 2 get classes for clz in top_level_classes(parse_ast(p).body): class_name = clz.name @@ -98,4 +98,3 @@ def gen_class_rst(class_name, writer): # 1.2 generate RST with open(f"{gendir}/{args.root}/{submodule_name}/{class_name}.rst", "w") as f_rst: gen_class_rst(f"{module_name}.{class_name}", f_rst) - diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index c9d3f5c6762..aa90909a641 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -25,7 +25,7 @@ for document in "${documents[@]}"; do -type f \ -name '*.ipynb' \ -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --to markdown \"{}\"" \; - + basedir=./${document} for md_file in `find "./${document}" -name "*.md"`; do filename=`basename ${md_file}` diff --git a/doc/vuepress/create_menu.py b/doc/vuepress/create_menu.py index 8ddd1c357e5..a689f2a55b0 100644 --- a/doc/vuepress/create_menu.py +++ b/doc/vuepress/create_menu.py @@ -54,7 +54,7 @@ def get_menubar_recursively(directory): for submodule in glob.glob(f"{doc}/**/") ]) }) - + # 1.2. Create Tools navbars.append({ 'text': "Tools", @@ -86,7 +86,7 @@ def get_menubar_recursively(directory): 'prefix': f"{doc}/", 'children': [f"README.md"] }) - + # 1.2.1. sort navbars[1]['children'].sort(key=lambda x: x['text'].lower()) navbars[2]['children'].sort(key=lambda x: x['text'].lower()) @@ -94,7 +94,7 @@ def get_menubar_recursively(directory): # 1.3 write navBars.yml with open('navbars.yml', 'w', encoding='utf-8') as f: yaml.dump(navbars, f, default_flow_style=False) - + # 2. Create sidebars # 2.1. Create guide sidebars.append({ @@ -123,4 +123,3 @@ def get_menubar_recursively(directory): # 2.3. Write sidebars.yml with open('sidebars.yml', 'w', encoding='utf-8') as f: yaml.dump(sidebars, f, default_flow_style=False) - diff --git a/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg b/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg index 5ab7d175fff..e35807efcd6 100644 --- a/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg +++ b/doc/vuepress/src/.vuepress/public/assets/icon/school_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg b/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg index c62c8e1af80..76d2603596e 100644 --- a/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg +++ b/doc/vuepress/src/.vuepress/public/assets/icon/speech_to_text_24dp_5F6368_FILL0_wght400_GRAD0_opsz24.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg b/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg index c27ede597a8..ba4156d0150 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/blog.svg b/doc/vuepress/src/.vuepress/public/assets/image/blog.svg index 00fc40d7926..896ddb3c811 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/blog.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/blog.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/box.svg b/doc/vuepress/src/.vuepress/public/assets/image/box.svg index 9e6408ed046..618eddab415 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/box.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/box.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/features.svg b/doc/vuepress/src/.vuepress/public/assets/image/features.svg index 6d62739369b..ceaded1de5d 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/features.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/features.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg b/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg index 37fa923df33..98d74c33f3f 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/github-dark.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg b/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg index d5e64918546..c679c236fd2 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/github-light.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/layout.svg b/doc/vuepress/src/.vuepress/public/assets/image/layout.svg index da754b58bed..d60be4cd53d 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/layout.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/layout.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg b/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg index 72056c9cdbc..28973388eb6 100644 --- a/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg +++ b/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/doc/vuepress/src/.vuepress/styles/index.scss b/doc/vuepress/src/.vuepress/styles/index.scss index 912abc5f90e..5e51c52ea74 100644 --- a/doc/vuepress/src/.vuepress/styles/index.scss +++ b/doc/vuepress/src/.vuepress/styles/index.scss @@ -41,4 +41,3 @@ body { font-size: 90%; opacity: 90%; } - diff --git a/doc/vuepress/src/.vuepress/styles/palette.scss b/doc/vuepress/src/.vuepress/styles/palette.scss index 01ecd52a13f..574b51c001f 100644 --- a/doc/vuepress/src/.vuepress/styles/palette.scss +++ b/doc/vuepress/src/.vuepress/styles/palette.scss @@ -1,3 +1,3 @@ // you can change colors here $theme-color: #096dd9; -$tip-bg-color: #F4FFF5; \ No newline at end of file +$tip-bg-color: #F4FFF5; From 336e8afd9e3120925e8ad7667359c8ca24a7dc0f Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Fri, 26 Jul 2024 18:36:05 +0900 Subject: [PATCH 03/57] Add and refactor docstrings by ChatGPT-4o-mini --- espnetez/config.py | 101 +++++++++++++++++++ espnetez/data/dump.py | 102 +++++++++++++++++-- espnetez/dataloader.py | 68 +++++++++++++ espnetez/dataset.py | 118 ++++++++++++++++++++++ espnetez/preprocess/sentencepiece.py | 90 ++++++++++++++--- espnetez/preprocess/tokenizer.py | 37 +++++++ espnetez/task.py | 69 +++++++++++++ espnetez/trainer.py | 142 ++++++++++++++++++++++++++- 8 files changed, 704 insertions(+), 23 deletions(-) diff --git a/espnetez/config.py b/espnetez/config.py index 88136165f16..d923f9ee520 100644 --- a/espnetez/config.py +++ b/espnetez/config.py @@ -4,6 +4,34 @@ def convert_none_to_None(dic): + """ + Recursively convert string representations of 'none' in a dictionary to Python's None. + + This function traverses a dictionary and replaces any occurrences of the string + "none" with the actual Python None type. If a value in the dictionary is another + dictionary, this function is called recursively to ensure all nested dictionaries + are processed. + + Args: + dic (dict): A dictionary potentially containing the string "none" as a value. + + Returns: + dict: The input dictionary with all instances of the string "none" replaced + by None. + + Examples: + >>> sample_dict = {'key1': 'none', 'key2': {'subkey1': 'none', 'subkey2': 'value'}} + >>> convert_none_to_None(sample_dict) + {'key1': None, 'key2': {'subkey1': None, 'subkey2': 'value'}} + + >>> nested_dict = {'level1': {'level2': {'level3': 'none'}}} + >>> convert_none_to_None(nested_dict) + {'level1': {'level2': {'level3': None}}} + + Note: + This function modifies the input dictionary in place, but it also returns + the modified dictionary for convenience. + """ for k, v in dic.items(): if isinstance(v, dict): dic[k] = convert_none_to_None(dic[k]) @@ -14,6 +42,38 @@ def convert_none_to_None(dic): def from_yaml(task, path): + """ + Load configuration from a YAML file and merge it with the default task configuration. + + This function reads a YAML configuration file from the specified path and merges + its contents with the default configuration for the specified task. If there are any + keys in the YAML file that have the string value "none", they are converted to Python's + `None` type. The resulting configuration dictionary is returned. + + Args: + task (str): The name of the task for which the configuration is being loaded. + path (str): The file path to the YAML configuration file. + + Returns: + dict: A dictionary containing the merged configuration settings. + + Raises: + FileNotFoundError: If the specified YAML file does not exist. + yaml.YAMLError: If the YAML file is not formatted correctly. + + Examples: + >>> config = from_yaml('speech_recognition', 'config.yaml') + >>> print(config) + {'learning_rate': 0.001, 'batch_size': 32, 'preprocessor_conf': None} + + >>> config = from_yaml('text_to_speech', 'path/to/config.yaml') + >>> print(config['model_type']) + 'tacotron2' + + Note: + Ensure that the task name provided corresponds to a valid task class + in the `espnetez.task` module to avoid runtime errors. + """ task_class = get_ez_task(task) with open(path, "r") as f: config = yaml.load(f, Loader=yaml.Loader) @@ -28,6 +88,47 @@ def from_yaml(task, path): def update_finetune_config(task, pretrain_config, path): + """ + Update the fine-tuning configuration with values from a specified YAML file. + + This function loads the fine-tuning configuration from a YAML file and + updates the provided pre-training configuration dictionary. It prioritizes + values from the fine-tuning configuration, while ensuring that any + distributed-related settings are reset to their defaults. Additionally, + it integrates default configurations from the specified task. + + Args: + task (str): The name of the task for which the configuration is being updated. + pretrain_config (dict): The existing pre-training configuration dictionary to be updated. + path (str): The file path to the YAML file containing the fine-tuning configuration. + + Returns: + dict: The updated pre-training configuration dictionary after merging with the fine-tuning + configuration and defaults from the specified task. + + Examples: + >>> pretrain_cfg = { + ... "learning_rate": 0.001, + ... "batch_size": 32, + ... "dist_backend": "nccl" + ... } + >>> updated_cfg = update_finetune_config("asr", pretrain_cfg, "finetune_config.yaml") + >>> print(updated_cfg) + { + "learning_rate": 0.0001, # updated from finetune_config.yaml + "batch_size": 32, + "dist_backend": "nccl", + "other_config": "default_value" # from task defaults + } + + Raises: + FileNotFoundError: If the specified YAML file does not exist. + yaml.YAMLError: If the YAML file is improperly formatted. + + Note: + The function assumes that the task class provides a method `get_default_config()` + which returns the default configuration as a dictionary. + """ with open(path, "r") as f: finetune_config = yaml.load(f, Loader=yaml.Loader) default_config = get_ez_task(task).get_default_config() diff --git a/espnetez/data/dump.py b/espnetez/data/dump.py index 5314a1373d8..90b22440a9b 100644 --- a/espnetez/data/dump.py +++ b/espnetez/data/dump.py @@ -9,13 +9,41 @@ def join_dumps( dump_prefix: List[str], output_dir: Union[str, Path], ): - """Create a joined dump file from a list of dump paths. + """ + Create a joined dump file from a list of dump paths. + + This function takes multiple dump paths and prefixes, reads the corresponding + dump files, and creates a new dump file in the specified output directory. + Each line from the original dump files is prefixed with the corresponding + prefix from the `dump_prefix` list. Args: - dump_paths (List[str]): List of paths for the dump directory. - dump_prefix (List[str]): List of prefixes for the dump files. - output_dir (Union[str, Path]): Output directory of the joined dump file. + dump_paths (List[str]): A list of paths for the dump directories. + Each path should contain the dump files to be joined. + dump_prefix (List[str]): A list of prefixes for the dump files. + Each prefix will be added to the beginning of the + corresponding lines in the joined output file. + output_dir (Union[str, Path]): The output directory where the joined dump + file will be saved. If the directory does + not exist, it will be created. + + Raises: + ValueError: If any of the expected dump files do not exist in the specified + dump paths. + + Examples: + >>> join_dumps( + ... dump_paths=["/path/to/dump1", "/path/to/dump2"], + ... dump_prefix=["dataset1", "dataset2"], + ... output_dir="/path/to/output" + ... ) + This will read dump files from "/path/to/dump1" and "/path/to/dump2", + prefix the lines with "dataset1-" and "dataset2-", and write the joined + content to "/path/to/output". + Note: + It is assumed that all dump directories contain the same set of dump file + names. If the dump files have different names, a ValueError will be raised. """ dump_file_names = [ os.path.basename(g) for g in glob.glob(os.path.join(dump_paths[0], "*")) @@ -45,14 +73,70 @@ def create_dump_file( dataset: Union[Dict[str, Dict], List[Dict]], data_inputs: Dict[str, Dict], ): - """Create a dump file for a dataset. + """ + Create a dump file for a dataset. + + This function generates a dump file in the specified directory containing + the specified data from the dataset. The dump file will include information + related to the input variables as specified in the `data_inputs` argument. Args: - dump_dir (str): Output folder of the dump files. - dataset (Union[Dict[str, Dict], List[Dict]]): Dictionary of dataset. - data_inputs (Dict[str, List[str, str]]): - data information for each input variables. + dump_dir (Union[str, Path]): + The output directory where the dump files will be saved. + If the directory does not exist, it will be created. + + dataset (Union[Dict[str, Dict], List[Dict]]): + The dataset from which to create the dump file. It can either be + a dictionary where each key represents a data entry or a list of + dictionaries representing multiple entries. + + data_inputs (Dict[str, Dict]): + A dictionary containing data information for each input variable. + Each key should correspond to a variable name, and the value should + be a list where the first element is the desired output file name + for that variable. + + Raises: + ValueError: + If `dataset` is neither a dictionary nor a list, or if any + expected data entry is missing. + + Examples: + Creating a dump file from a dictionary dataset: + + >>> dump_dir = "output/dump" + >>> dataset = { + ... 0: {"feature1": "value1", "feature2": "value2"}, + ... 1: {"feature1": "value3", "feature2": "value4"}, + ... } + >>> data_inputs = { + ... "feature1": ["feature1_dump.txt"], + ... "feature2": ["feature2_dump.txt"], + ... } + >>> create_dump_file(dump_dir, dataset, data_inputs) + + This will create two files: `feature1_dump.txt` and `feature2_dump.txt` + in the `output/dump` directory, each containing the corresponding data. + + Creating a dump file from a list dataset: + + >>> dump_dir = "output/dump" + >>> dataset = [ + ... {"feature1": "value1", "feature2": "value2"}, + ... {"feature1": "value3", "feature2": "value4"}, + ... ] + >>> data_inputs = { + ... "feature1": ["feature1_dump.txt"], + ... "feature2": ["feature2_dump.txt"], + ... } + >>> create_dump_file(dump_dir, dataset, data_inputs) + + Similar to the previous example, this will create the same dump files + in the specified output directory. + Note: + Ensure that the output directory has the necessary write permissions + to avoid any I/O errors during file creation. """ if not os.path.exists(dump_dir): os.makedirs(dump_dir) diff --git a/espnetez/dataloader.py b/espnetez/dataloader.py index db78d92d7ea..745112e3d87 100644 --- a/espnetez/dataloader.py +++ b/espnetez/dataloader.py @@ -4,8 +4,76 @@ class Dataloader(AbsIterFactory): + """ + DataLoader class for building data iterators. + + This class extends the `AbsIterFactory` and provides a method to create + a PyTorch DataLoader. It accepts keyword arguments that can be passed + directly to the DataLoader constructor. + + Attributes: + kwargs (dict): A dictionary of keyword arguments for the DataLoader. + + Args: + **kwargs: Arbitrary keyword arguments that are passed to the + `torch.utils.data.DataLoader` constructor. Common arguments include: + - dataset: The dataset from which to load the data. + - batch_size: Number of samples per batch. + - shuffle: Whether to shuffle the data at every epoch. + - num_workers: Number of subprocesses to use for data loading. + - collate_fn: A function to merge a list of samples to form a + mini-batch. + + Examples: + >>> from torchvision import datasets, transforms + >>> transform = transforms.Compose([transforms.ToTensor()]) + >>> dataset = datasets.MNIST(root='./data', train=True, + ... transform=transform, download=True) + >>> dataloader = Dataloader(dataset=dataset, batch_size=32, + ... shuffle=True) + >>> train_loader = dataloader.build_iter(epoch=1) + + Note: + The `build_iter` method does not utilize the `epoch` and `shuffle` + parameters in its current implementation. They are included for + potential future use, such as enabling shuffling based on epoch + number. + """ + def __init__(self, **kwargs): self.kwargs = kwargs def build_iter(self, epoch: int, shuffle: bool = None) -> DataLoader: + """ + Constructs a PyTorch DataLoader instance. + + This method initializes and returns a DataLoader using the + parameters provided during the instantiation of the Dataloader + class. It can be customized with various options to suit the + needs of the data loading process. + + Args: + epoch (int): The current epoch number. This can be used + to implement epoch-based behaviors, such as shuffling + the dataset. + shuffle (bool, optional): A flag indicating whether to + shuffle the dataset. If set to True, the data will be + shuffled before each epoch. Defaults to None, which + means the value set in kwargs will be used. + + Returns: + DataLoader: A PyTorch DataLoader instance configured with + the provided arguments. + + Examples: + >>> dataloader = Dataloader(batch_size=32, dataset=my_dataset) + >>> train_loader = dataloader.build_iter(epoch=1, shuffle=True) + >>> for data in train_loader: + ... # Process data + ... pass + + Note: + Ensure that the dataset provided in kwargs is compatible + with the DataLoader's expected format. + """ return DataLoader(**self.kwargs) diff --git a/espnetez/dataset.py b/espnetez/dataset.py index d31b92c67c4..4a1a9c73450 100644 --- a/espnetez/dataset.py +++ b/espnetez/dataset.py @@ -4,14 +4,132 @@ class ESPnetEZDataset(AbsDataset): + """ + A dataset class for handling ESPnet data with easy access to data information. + + This class extends the AbsDataset class and provides functionalities to + manage a dataset and its associated metadata. It allows users to retrieve + dataset items using unique identifiers and check for available names in + the dataset. + + Attributes: + dataset (Union[list, Tuple]): The dataset containing the actual data entries. + data_info (Dict[str, callable]): A dictionary mapping attribute names to + functions that extract those attributes from the dataset. + + Args: + dataset (Union[list, Tuple]): The dataset from which data will be extracted. + data_info (Dict[str, callable]): A dictionary where keys are attribute names + and values are functions that process the dataset entries. + + Methods: + has_name(name): Checks if the given name exists in the data_info dictionary. + names() -> Tuple[str, ...]: Returns a tuple of all names in the data_info. + __getitem__(uid: Union[str, int]) -> Tuple[str, Dict]: Retrieves the data + entry corresponding to the provided unique identifier. + __len__() -> int: Returns the total number of entries in the dataset. + + Examples: + >>> dataset = [("audio1.wav", "transcription1"), ("audio2.wav", "transcription2")] + >>> data_info = { + ... "audio": lambda x: x[0], + ... "transcription": lambda x: x[1] + ... } + >>> ez_dataset = ESPnetEZDataset(dataset, data_info) + >>> ez_dataset.has_name("audio") + True + >>> ez_dataset.names() + ('audio', 'transcription') + >>> ez_dataset[0] + ('0', {'audio': 'audio1.wav', 'transcription': 'transcription1'}) + >>> len(ez_dataset) + 2 + + Note: + The dataset and data_info must be provided in a compatible format to ensure + proper functionality of the methods. + """ + def __init__(self, dataset, data_info): self.dataset = dataset self.data_info = data_info def has_name(self, name) -> bool: + """ + Check if the specified name exists in the dataset's data information. + + This method searches the `data_info` attribute of the dataset to determine + if the given `name` is present as a key. It is useful for validating + whether certain attributes or features are available in the dataset. + + Args: + name (str): The name to search for in the dataset's data information. + + Returns: + bool: True if the name exists in the data information; False otherwise. + + Examples: + >>> dataset = ESPnetEZDataset(dataset=[...], data_info={'feature1': ..., 'feature2': ...}) + >>> dataset.has_name('feature1') + True + >>> dataset.has_name('feature3') + False + + Note: + The method performs a simple membership check using the `in` operator, + which is efficient for dictionaries. + """ return name in self.data_info def names(self) -> Tuple[str, ...]: + """ + A dataset class for ESPnet that handles data retrieval and management. + + This class extends the abstract dataset class to provide functionalities + specific to the ESPnet framework. It manages a dataset and its associated + metadata, allowing for efficient data access and manipulation. + + Attributes: + dataset (Union[list, tuple]): The underlying dataset that contains the data entries. + data_info (Dict[str, callable]): A dictionary mapping names to functions that process + each data entry in the dataset. + + Args: + dataset (Union[list, tuple]): The dataset to be wrapped. + data_info (Dict[str, callable]): A dictionary where keys are the names of the data + attributes and values are functions that extract or transform the data from the + dataset. + + Methods: + has_name(name: str) -> bool: + Checks if a given name exists in the data_info. + + names() -> Tuple[str, ...]: + Returns a tuple of all the names available in the data_info. + + __getitem__(uid: Union[str, int]) -> Tuple[str, Dict]: + Retrieves the data entry corresponding to the provided identifier. + + __len__() -> int: + Returns the number of entries in the dataset. + + Examples: + >>> dataset = ESPnetEZDataset(dataset=[...], data_info={'feature': lambda x: x.feature, + ... 'label': lambda x: x.label}) + >>> dataset.has_name('feature') + True + >>> dataset.names() + ('feature', 'label') + >>> entry = dataset[0] + >>> print(entry) + ('0', {'feature': ..., 'label': ...}) + >>> len(dataset) + 100 + + Note: + The functions provided in the data_info should be callable and should accept a single + argument corresponding to an entry from the dataset. + """ return tuple(self.data_info.keys()) def __getitem__(self, uid: Union[str, int]) -> Tuple[str, Dict]: diff --git a/espnetez/preprocess/sentencepiece.py b/espnetez/preprocess/sentencepiece.py index e74522256e8..0e784e72291 100644 --- a/espnetez/preprocess/sentencepiece.py +++ b/espnetez/preprocess/sentencepiece.py @@ -12,12 +12,44 @@ def prepare_sentences( remove_characters: str = "", ): """ - Create train.txt file for sentencepiece training from the given dump file. + Create a training text file for SentencePiece model training from the + provided dump text files. + + This function consolidates multiple text files into a single `train.txt` + file, which is formatted for use in SentencePiece training. It also + provides an option to remove specified characters from the text before + writing to the output file. Args: - dump_text_paths (Union[str, Path]): Dump text file path. - output_path (Union[str, Path]): Output directory for train.txt file. - remove_characters (str): Characters to be removed from the text. + dump_text_paths (Union[str, Path]): + A single dump text file path or a list of paths to the dump + text files that will be processed. + output_path (Union[str, Path]): + The directory where the `train.txt` file will be saved. + If the directory does not exist, it will be created. + remove_characters (str, optional): + A string containing characters to be removed from the text. + Defaults to an empty string, meaning no characters will be + removed. + + Raises: + FileNotFoundError: If any of the dump text files do not exist. + IOError: If there is an error reading from the dump text files + or writing to the output path. + + Examples: + >>> prepare_sentences("data/dump.txt", "output", remove_characters=",.!") + This will create an `output/train.txt` file from `data/dump.txt`, + removing commas, periods, and exclamation marks from the text. + + >>> prepare_sentences(["data/dump1.txt", "data/dump2.txt"], "output") + This will create an `output/train.txt` file by concatenating + `data/dump1.txt` and `data/dump2.txt` without removing any characters. + + Note: + Ensure that the input dump text files are properly formatted, as + the function expects each line to have a space-separated format + where the text to be processed is after the first space. """ # Please join the dump set before running this function. if not os.path.exists(output_path): @@ -48,17 +80,49 @@ def train_sentencepiece( model_type: str = "bpe", user_defined_symbols: list = [], ): - """Main function to train sentencepiece model. + """ + Main function to train a SentencePiece model. + + This function trains a SentencePiece model using the provided training + data and saves the resulting model and vocabulary files to the specified + output directory. The model can be customized through various parameters + such as vocabulary size, character coverage, and model type. Args: - dump_text_path (Union[str, Path]): Path to the train.txt file. - output_path (Union[str, Path]): Output directory to store - sentencepiece model and vocaburary list. - vocab_size (int, optional): Vocaburary size. Defaults to 5000. - character_coverage (float, optional): Character coverage. - Defaults to 0.9995. - model_type (str, optional): Model type of sentencepiece. Defaults to "bpe". - user_defined_symbols (list, optional): User defined symbols. + dump_text_path (Union[str, Path]): Path to the `train.txt` file + containing the training data for the SentencePiece model. + output_path (Union[str, Path]): Output directory where the trained + SentencePiece model and vocabulary list will be stored. + vocab_size (int, optional): The size of the vocabulary to be generated + by the SentencePiece model. Defaults to 5000. + character_coverage (float, optional): The character coverage rate + for the model, which indicates the percentage of characters in + the training data that should be covered. Defaults to 0.9995. + model_type (str, optional): The type of model to be trained. + Options include 'bpe' (Byte Pair Encoding), 'unigram', + 'char', and 'word'. Defaults to "bpe". + user_defined_symbols (list, optional): A list of user-defined symbols + that should be included in the model. Defaults to an empty list. + + Raises: + FileNotFoundError: If the specified `dump_text_path` does not exist. + Exception: If the training of the SentencePiece model fails for any + reason. + + Examples: + >>> train_sentencepiece( + ... dump_text_path='path/to/train.txt', + ... output_path='path/to/output', + ... vocab_size=8000, + ... character_coverage=0.995, + ... model_type='unigram', + ... user_defined_symbols=['', ''] + ... ) + + Note: + Ensure that the `train.txt` file has been prepared using the + `prepare_sentences` function before calling this function. + The output directory will be created if it does not already exist. """ # Please prepare sentences before running this function. spm.SentencePieceTrainer.Train( diff --git a/espnetez/preprocess/tokenizer.py b/espnetez/preprocess/tokenizer.py index 32093497df7..47376404c4b 100644 --- a/espnetez/preprocess/tokenizer.py +++ b/espnetez/preprocess/tokenizer.py @@ -11,6 +11,43 @@ def tokenize( sos_eos="", **kwargs, ): + """ + Tokenizes the input text and saves the output to a specified file. + + This function utilizes the ESPnet tokenizer to process a given input text file, + tokenize its contents, and save the results to an output file. Additionally, + it can optionally write a vocabulary file and allows for the inclusion of + special symbols such as blank tokens, out-of-vocabulary tokens, and start/end + of sentence tokens. + + Args: + input (str): Path to the input text file to be tokenized. + output (str): Path to the output file where the tokenized text will be saved. + write_vocabulary (bool, optional): Whether to write the vocabulary to a file. + Defaults to True. + blank (str, optional): Symbol to represent blank tokens. Defaults to "". + oov (str, optional): Symbol to represent out-of-vocabulary tokens. + Defaults to "". + sos_eos (str, optional): Symbol to represent start and end of sentence tokens. + Defaults to "". + **kwargs: Additional keyword arguments to customize the tokenizer behavior. + + Returns: + None: The function does not return any value but writes the tokenized + output to the specified file. + + Raises: + FileNotFoundError: If the input file does not exist. + ValueError: If the provided arguments are invalid. + + Examples: + # Basic usage + tokenize('input.txt', 'output.txt') + + # Customizing the tokenizer with additional symbols + tokenize('input.txt', 'output.txt', write_vocabulary=False, + blank="", oov="", sos_eos="") + """ parser = get_parser() kws = ["--input", input, "--output", output] kws += [ diff --git a/espnetez/task.py b/espnetez/task.py index 2965bd933e4..8d8638d07b6 100644 --- a/espnetez/task.py +++ b/espnetez/task.py @@ -66,6 +66,44 @@ def get_ez_task(task_name: str, use_custom_dataset: bool = False) -> AbsTask: + """ + Retrieve a customized task class for the ESPnet-EZ framework. + + This function returns a task class based on the specified task name. + If the `use_custom_dataset` flag is set to True, a version of the task + class that supports custom datasets will be returned. The returned class + inherits from the appropriate base task class and may be extended with + additional functionality. + + Args: + task_name (str): The name of the task to retrieve. This must be one of + the keys defined in the `TASK_CLASSES` dictionary, such as 'asr', + 'mt', 'tts', etc. + use_custom_dataset (bool, optional): A flag indicating whether to use + a version of the task class that supports custom datasets. Defaults + to False. + + Returns: + AbsTask: An instance of the task class corresponding to the provided + `task_name`. If `use_custom_dataset` is True, the returned class will + be capable of handling custom datasets. + + Raises: + KeyError: If `task_name` is not found in the `TASK_CLASSES` dictionary. + + Examples: + >>> asr_task = get_ez_task("asr") + >>> custom_asr_task = get_ez_task("asr", use_custom_dataset=True) + + >>> mt_task = get_ez_task("mt") + >>> custom_mt_task = get_ez_task("mt", use_custom_dataset=True) + + Note: + The task classes are designed to be used within the ESPnet-EZ framework, + which allows for flexibility in handling various speech and language tasks. + Ensure that the required dependencies for the specific task are properly + installed and configured. + """ task_class = TASK_CLASSES[task_name] if use_custom_dataset: @@ -85,6 +123,37 @@ def build_model(cls, args=None): def get_ez_task_with_dataset(task_name: str) -> AbsTask: + """ + Create an ESPnet-EZ task class with a custom dataset for a given task. + + This function returns a task class that inherits from the specified + task class in the ESPnet framework, enabling the use of custom datasets + for training and validation. The created task class includes methods + for building models and iterators specifically tailored for handling + datasets. + + Args: + task_name (str): The name of the task for which the class is being created. + This should correspond to one of the predefined task classes + in the ESPnet framework, such as 'asr', 'tts', etc. + + Returns: + AbsTask: A subclass of AbsTask that supports custom datasets for the specified task. + + Examples: + >>> from espnetez.task import get_ez_task_with_dataset + >>> custom_asr_task = get_ez_task_with_dataset("asr") + >>> custom_asr_task.train_dataset = my_custom_train_dataset + >>> custom_asr_task.valid_dataset = my_custom_valid_dataset + >>> model = custom_asr_task.build_model(args) + >>> iterator = custom_asr_task.build_iter_factory(args, distributed_option, mode='train') + + Note: + Ensure that the specified task name is valid and that the corresponding + task class is available in the TASK_CLASSES dictionary. The created + task class will need to have its `train_dataset` and `valid_dataset` + attributes set to the appropriate dataset instances before training. + """ task_class = TASK_CLASSES[task_name] class ESPnetEZDataTask(task_class): diff --git a/espnetez/trainer.py b/espnetez/trainer.py index 6e433ed7451..09f885d0c03 100644 --- a/espnetez/trainer.py +++ b/espnetez/trainer.py @@ -13,6 +13,51 @@ def check_argument( train_dataloader, valid_dataloader, ): + """ + Validate the arguments for training and validation data sources. + + This function checks the consistency of the input arguments used for + specifying training and validation data sources. It ensures that + the user adheres to the rules of specifying either dump directories, + datasets, or dataloaders, but not a mix of them. The function will + raise a ValueError if the conditions are not met. + + Args: + train_dump_dir (str or None): The directory containing training + dump files. Should be None if using datasets or dataloaders. + valid_dump_dir (str or None): The directory containing validation + dump files. Should be None if using datasets or dataloaders. + train_dataset (Dataset or None): The training dataset. Should be + None if using dump files or dataloaders. + valid_dataset (Dataset or None): The validation dataset. Should be + None if using dump files or dataloaders. + train_dataloader (DataLoader or None): The training dataloader. + Should be None if using dump files or datasets. + valid_dataloader (DataLoader or None): The validation dataloader. + Should be None if using dump files or datasets. + + Returns: + bool: Returns True if all checks pass. + + Raises: + ValueError: If any of the argument conditions are violated. + + Examples: + # Example of valid usage with dump files + check_argument('/path/to/train_dump', '/path/to/valid_dump', None, None, None, None) + + # Example of valid usage with datasets + check_argument(None, None, train_dataset, valid_dataset, None, None) + + # Example of invalid usage - mix of dump files and datasets + check_argument('/path/to/train_dump', None, train_dataset, None, None, None) + # Raises ValueError: If you try to use dump file, dataset or dataloader should be None. + + Note: + Ensure to specify at least one of the arguments: dump directories, + datasets, or dataloaders. The function enforces exclusive use of + each data source type. + """ # if we have dump files, dataset/dataloader should be None, # else if we have dataset, dumpfile/dataloader should be None, # else if we have dataloader, dumpfile/dataset should be None. @@ -69,7 +114,55 @@ def check_argument( class Trainer: - """Generic trainer class for ESPnet training!""" + """ + Generic trainer class for ESPnet training. + + This class is responsible for managing the training process of ESPnet models. + It handles the configuration, dataset preparation, and the training loop. + The Trainer class supports multiple input methods including dump directories, + custom datasets, and dataloaders. It ensures that the provided arguments are + consistent and valid before starting the training process. + + Attributes: + train_config (Namespace): Configuration for training, can be a dictionary or Namespace object. + task_class (Task): Task class instantiated from the provided task identifier. + stats_dir (str): Directory where statistics for training and validation will be stored. + output_dir (str): Directory where model outputs will be saved. + + Args: + task (str): The task identifier used to retrieve the corresponding task class. + train_config (Union[dict, Namespace]): Configuration for training. + output_dir (str): Directory for saving model outputs. + stats_dir (str): Directory for storing training statistics. + data_info (dict, optional): Information about the dataset paths and types. + train_dump_dir (str, optional): Directory containing training dump files. + valid_dump_dir (str, optional): Directory containing validation dump files. + train_dataset (Dataset, optional): Custom training dataset. + valid_dataset (Dataset, optional): Custom validation dataset. + train_dataloader (DataLoader, optional): DataLoader for training data. + valid_dataloader (DataLoader, optional): DataLoader for validation data. + build_model_fn (callable, optional): Function to build the model. + **kwargs: Additional keyword arguments for configuring the training. + + Raises: + ValueError: If any of the argument validation checks fail. + + Examples: + >>> trainer = Trainer( + ... task='asr', + ... train_config={'batch_size': 32, 'learning_rate': 0.001}, + ... output_dir='./output', + ... stats_dir='./stats', + ... train_dump_dir='./train_dump', + ... valid_dump_dir='./valid_dump' + ... ) + >>> trainer.collect_stats() # Collect statistics from the dataset + >>> trainer.train() # Start the training process + + Note: + Ensure that either dump directories, datasets, or dataloaders are specified + as input parameters, but not a combination of them in conflicting ways. + """ def __init__( self, @@ -143,6 +236,24 @@ def __init__( self.task_class.build_model_fn = build_model_fn def train(self): + """ + Train the model using the specified training configuration. + + This method orchestrates the training process by first ensuring that + the necessary shape files are available. It checks for the presence + of shape files in the specified statistics directory, and if they + are found, it proceeds to invoke the main training routine of the + task class. + + Raises: + AssertionError: If no shape files are found in the statistics + directory for either training or validation. + + Examples: + >>> trainer = Trainer(task='my_task', train_config=my_train_config, + ... output_dir='output/', stats_dir='stats/') + >>> trainer.train() # Starts the training process + """ # after collect_stats, define shape files self.train_config.train_shape_file = glob.glob( os.path.join(self.stats_dir, "train", "*shape") @@ -161,6 +272,35 @@ def train(self): self.task_class.main(self.train_config) def collect_stats(self): + """ + Collects statistics for training and validation datasets. + + This method initializes the process of gathering statistical data + from the training and validation datasets. It creates the necessary + directories to store the statistics if they do not already exist + and sets the configuration parameters for collecting statistics. + The statistics are used to define the shape files required for + training. + + The method will call the `main` function of the `task_class` + with the updated configuration, which includes the output directory + set to the statistics directory. + + Raises: + OSError: If the directory for storing statistics cannot be created. + + Examples: + >>> trainer = Trainer(task='example_task', train_config=some_config, + ... output_dir='/path/to/output', stats_dir='/path/to/stats') + >>> trainer.collect_stats() + + Note: + This method must be called before training to ensure that + the shape files are defined properly. After running this method, + the `train_shape_file` and `valid_shape_file` attributes + of `train_config` will be populated based on the collected + statistics. + """ if not os.path.exists(self.stats_dir): os.makedirs(self.stats_dir) From 2c761d5adcdf67aa31c7b85c93bb1ba48728596f Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Sat, 27 Jul 2024 09:38:21 +0900 Subject: [PATCH 04/57] Add or reformat docstrings by LLM --- espnet2/asr/bayes_risk_ctc.py | 218 ++- espnet2/asr/ctc.py | 178 ++- espnet2/asr/decoder/abs_decoder.py | 59 + .../hugging_face_transformers_decoder.py | 201 ++- espnet2/asr/decoder/mlm_decoder.py | 90 +- espnet2/asr/decoder/rnn_decoder.py | 262 ++++ espnet2/asr/decoder/s4_decoder.py | 168 ++- espnet2/asr/decoder/transducer_decoder.py | 292 +++- espnet2/asr/decoder/transformer_decoder.py | 508 +++++-- espnet2/asr/decoder/whisper_decoder.py | 234 ++- espnet2/asr/discrete_asr_espnet_model.py | 71 +- espnet2/asr/encoder/abs_encoder.py | 93 ++ espnet2/asr/encoder/branchformer_encoder.py | 120 +- espnet2/asr/encoder/conformer_encoder.py | 97 +- .../contextual_block_conformer_encoder.py | 1271 +++++++++-------- .../contextual_block_transformer_encoder.py | 108 +- espnet2/asr/encoder/e_branchformer_encoder.py | 144 +- espnet2/asr/encoder/hubert_encoder.py | 405 ++++-- .../hugging_face_transformers_encoder.py | 97 +- espnet2/asr/encoder/linear_encoder.py | 90 +- espnet2/asr/encoder/longformer_encoder.py | 113 +- espnet2/asr/encoder/rnn_encoder.py | 85 +- espnet2/asr/encoder/transformer_encoder.py | 102 +- .../encoder/transformer_encoder_multispkr.py | 110 +- espnet2/asr/encoder/vgg_rnn_encoder.py | 76 +- espnet2/asr/encoder/wav2vec2_encoder.py | 135 +- espnet2/asr/encoder/whisper_encoder.py | 181 ++- espnet2/asr/espnet_model.py | 157 +- espnet2/asr/frontend/abs_frontend.py | 74 + espnet2/asr/frontend/asteroid_frontend.py | 93 +- espnet2/asr/frontend/default.py | 99 +- espnet2/asr/frontend/fused.py | 79 + espnet2/asr/frontend/melspec_torch.py | 75 +- espnet2/asr/frontend/s3prl.py | 94 +- espnet2/asr/frontend/whisper.py | 134 +- espnet2/asr/frontend/windowing.py | 88 +- espnet2/asr/layers/cgmlp.py | 164 ++- espnet2/asr/layers/fastformer.py | 125 +- espnet2/asr/maskctc_model.py | 215 ++- espnet2/asr/pit_espnet_model.py | 234 ++- espnet2/asr/postencoder/abs_postencoder.py | 66 + .../hugging_face_transformers_postencoder.py | 92 +- .../postencoder/length_adaptor_postencoder.py | 74 +- espnet2/asr/preencoder/abs_preencoder.py | 83 ++ espnet2/asr/preencoder/linear.py | 76 +- espnet2/asr/preencoder/sinc.py | 169 ++- espnet2/asr/specaug/abs_specaug.py | 54 +- espnet2/asr/specaug/specaug.py | 64 +- espnet2/asr/state_spaces/attention.py | 186 ++- espnet2/asr/state_spaces/base.py | 537 ++++++- espnet2/asr/state_spaces/block.py | 156 +- espnet2/asr/state_spaces/cauchy.py | 231 ++- espnet2/asr/state_spaces/components.py | 589 +++++++- espnet2/asr/state_spaces/ff.py | 88 ++ espnet2/asr/state_spaces/model.py | 194 ++- espnet2/asr/state_spaces/pool.py | 653 ++++++++- espnet2/asr/state_spaces/residual.py | 293 +++- espnet2/asr/state_spaces/s4.py | 871 ++++++++++- espnet2/asr/state_spaces/utils.py | 247 +++- .../asr/transducer/beam_search_transducer.py | 22 +- espnet2/asr/transducer/error_calculator.py | 170 ++- .../asr/transducer/rnnt_multi_blank/rnnt.py | 149 +- .../rnnt_multi_blank/rnnt_multi_blank.py | 516 +++++-- 63 files changed, 10838 insertions(+), 1881 deletions(-) diff --git a/espnet2/asr/bayes_risk_ctc.py b/espnet2/asr/bayes_risk_ctc.py index bc415d95df0..7554a7941b7 100644 --- a/espnet2/asr/bayes_risk_ctc.py +++ b/espnet2/asr/bayes_risk_ctc.py @@ -9,6 +9,37 @@ class BayesRiskCTC(torch.nn.Module): + """ + Implements the Bayes Risk Connectionist Temporal Classification (BRCTC) loss. + + This class extends torch.nn.Module to provide a BRCTC loss implementation + based on the paper "Bayes Risk CTC: Efficient Sequence Labelling with Neural Networks" + (https://openreview.net/forum?id=Bd7GueaTxUz). + + BRCTC introduces a risk-based approach to CTC, allowing for more flexible + and potentially improved sequence labeling in tasks such as speech recognition. + + Attributes: + risk_strategy (str): Strategy for calculating risk. Can be 'exp' or 'exp_rel'. + group_strategy (str): Strategy for grouping. Can be 'end' or 'end_mean'. + risk_factor (float): Factor controlling the influence of the risk in the loss calculation. + + Args: + risk_strategy (str, optional): Risk calculation strategy. Defaults to 'exp'. + group_strategy (str, optional): Grouping strategy. Defaults to 'end'. + risk_factor (float, optional): Risk factor. Defaults to 0.0. + + Raises: + AssertionError: If an invalid risk_strategy or group_strategy is provided. + + Note: + This implementation requires the K2 library for finite-state automata operations. + + Example: + >>> brctc = BayesRiskCTC(risk_strategy='exp', group_strategy='end', risk_factor=0.1) + >>> loss = brctc(nnet_output, ys_pad, hlens, ylens) + """ + def __init__(self, risk_strategy="exp", group_strategy="end", risk_factor=0.0): super().__init__() @@ -20,6 +51,39 @@ def __init__(self, risk_strategy="exp", group_strategy="end", risk_factor=0.0): self.risk_factor = risk_factor def forward(self, nnet_output, ys_pad, hlens, ylens): + """ + Compute the Bayes Risk CTC loss for a batch of sequences. + + This method implements the forward pass of the Bayes Risk CTC loss calculation. + It handles reordering and filtering of input sequences, and delegates the core + loss computation to the forward_core method. + + Args: + nnet_output (torch.Tensor): Output from the neural network, shape (B, T, C), + where B is batch size, T is the maximum sequence length, and C is the number of classes. + ys_pad (torch.Tensor): Padded target sequences, shape (B, U), + where U is the maximum target sequence length. + hlens (torch.Tensor): Lengths of input sequences, shape (B,). + ylens (torch.Tensor): Lengths of target sequences, shape (B,). + + Returns: + torch.Tensor: The computed loss for each sequence in the batch, shape (B,). + + Note: + This method reorders the input sequences based on their lengths (hlens) in descending order, + and filters out invalid examples where the input length is smaller than the minimum required length. + + Raises: + Warning: If all examples in the batch are invalid for Bayes Risk CTC calculation. + + Example: + >>> brctc = BayesRiskCTC() + >>> nnet_output = torch.randn(32, 100, 50) # (batch_size, max_time, num_classes) + >>> ys_pad = torch.randint(0, 50, (32, 20)) # (batch_size, max_target_length) + >>> hlens = torch.randint(80, 101, (32,)) # Input lengths + >>> ylens = torch.randint(15, 21, (32,)) # Target lengths + >>> loss = brctc(nnet_output, ys_pad, hlens, ylens) + """ # Reorder and filter out invalid examples: # A. K2 requires that hlens are in descending order; # B. remove all examples whose hlens is smaller than necessary. @@ -47,6 +111,38 @@ def forward(self, nnet_output, ys_pad, hlens, ylens): return loss_utt def forward_core(self, nnet_output, ys, hlens, ylens): + """ + Compute the core Bayes Risk CTC loss for a batch of sequences. + + This method implements the main logic of the Bayes Risk CTC loss calculation, + including building the CTC graphs, performing intersections, and computing + forward-backward scores. + + Args: + nnet_output (torch.Tensor): Output from the neural network, shape (B, T, C), + where B is batch size, T is the maximum sequence length, and C is the number of classes. + ys (List[List[int]]): List of target label sequences (not padded). + hlens (torch.Tensor): Lengths of input sequences, shape (B,). + ylens (torch.Tensor): Lengths of target sequences, shape (B,). + + Returns: + torch.Tensor: The computed loss for each sequence in the batch, shape (B,). + + Note: + This method uses the K2 library for finite-state automata operations and + implements the core algorithm of Bayes Risk CTC as described in the paper. + + Raises: + NotImplementedError: If an unsupported group_strategy is specified. + + Example: + >>> brctc = BayesRiskCTC() + >>> nnet_output = torch.randn(32, 100, 50) # (batch_size, max_time, num_classes) + >>> ys = [[1, 2, 3], [4, 5, 6, 7], ...] # List of target sequences + >>> hlens = torch.randint(80, 101, (32,)) # Input lengths + >>> ylens = torch.tensor([len(y) for y in ys]) # Target lengths + >>> loss = brctc.forward_core(nnet_output, ys, hlens, ylens) + """ # (1) Find the shape (B, T, _), U = nnet_output.size(), max(ylens) @@ -147,7 +243,36 @@ def forward_core(self, nnet_output, ys, hlens, ylens): raise NotImplementedError def get_risk_scores(self, loss_state, hlens, risk_factor): - """Add the bayes risk in multiple ways""" + """ + Calculate the Bayes risk scores for each state in the lattice. + + This method computes risk scores based on the specified risk strategy, + which are then used to modify the loss calculation in the Bayes Risk CTC algorithm. + + Args: + loss_state (torch.Tensor): The loss state tensor, shape (B, U, T), + where B is batch size, U is the number of unique labels, + and T is the maximum sequence length. + hlens (torch.Tensor): Lengths of input sequences, shape (B,). + risk_factor (float): The risk factor to scale the computed risk. + + Returns: + torch.Tensor: The computed risk scores, shape (B, U, T). + + Raises: + NotImplementedError: If an unsupported risk_strategy is specified. + + Note: + The risk calculation depends on the risk_strategy attribute: + - 'exp': Exponential risk based on the position in the sequence. + - 'exp_rel': Exponential risk relative to the position of maximum loss. + + Example: + >>> brctc = BayesRiskCTC(risk_strategy='exp', risk_factor=0.1) + >>> loss_state = torch.randn(32, 20, 100) # (batch_size, num_labels, max_time) + >>> hlens = torch.randint(80, 101, (32,)) # Input lengths + >>> risk_scores = brctc.get_risk_scores(loss_state, hlens, 0.1) + """ B, U, T = loss_state.size() if self.risk_strategy == "exp": @@ -177,6 +302,40 @@ def get_risk_scores(self, loss_state, hlens, risk_factor): def find_all_index( self, ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, arc_map_b ): + """ + Find indices for arcs and states in the lattice. + + This method computes various indices for arcs and states in the CTC lattice, + which are used in the forward-backward algorithm of the Bayes Risk CTC loss calculation. + + Args: + ragged_lat (k2.RaggedTensor): The ragged lattice tensor. + ctc_graph (k2.Fsa): The CTC graph. + dense_fsa_vec (k2.DenseFsaVec): The dense FSA vector. + arc_map_a (torch.Tensor): Arc map for the first FSA. + arc_map_b (torch.Tensor): Arc map for the second FSA. + + Returns: + Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]]: + A tuple containing two tuples: + 1. Arc indices: (arc_u_idx, arc_t_idx, arc_k_idx, arc_b_idx) + 2. State indices: (state_u_idx, state_t_idx, state_k_idx, state_b_idx) + + Where: + - u_idx: label indices + - t_idx: time indices + - k_idx: class indices + - b_idx: batch indices + + Note: + This method handles the indexing for both arcs and states in the lattice, + including special handling for start and end states. + + Example: + >>> brctc = BayesRiskCTC() + >>> # Assume ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, and arc_map_b are properly initialized + >>> arc_indices, state_indices = brctc.find_all_index(ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, arc_map_b) + """ # This function finds the index of (b, t, u, d) for each arc and each state num_fsas = len(ctc_graph.arcs.row_splits(1)) - 1 @@ -239,6 +398,37 @@ def find_all_index( ) def find_minimum_hlens(self, ys_pad, ylens): + """ + Calculate the minimum possible input lengths for each sequence in the batch. + + This method computes the minimum required input length for each target sequence, + considering the CTC alignment rules. It also removes padding from the target sequences. + + Args: + ys_pad (torch.Tensor): Padded target sequences, shape (B, U), + where B is batch size and U is the maximum target sequence length. + ylens (torch.Tensor): Lengths of target sequences, shape (B,). + + Returns: + Tuple[List[List[int]], torch.Tensor]: + - List[List[int]]: Unpadded target sequences. + - torch.Tensor: Minimum required input lengths for each sequence, shape (B,). + + Note: + The minimum input length for a CTC alignment is calculated by considering + that each label requires at least one frame, and consecutive identical + labels require an additional frame between them. + + Example: + >>> brctc = BayesRiskCTC() + >>> ys_pad = torch.tensor([[1, 2, 2, 3, 0], [4, 4, 5, 0, 0]]) + >>> ylens = torch.tensor([4, 3]) + >>> ys, min_hlens = brctc.find_minimum_hlens(ys_pad, ylens) + >>> print(ys) + [[1, 2, 2, 3], [4, 4, 5]] + >>> print(min_hlens) + tensor([5, 4]) + """ device = ys_pad.device ys_pad, ylens = ys_pad.cpu().tolist(), ylens.cpu().tolist() ys, min_hlens = [], [] @@ -265,7 +455,31 @@ def find_minimum_hlens(self, ys_pad, ylens): def log_substraction_exp(a, b): - """return (a.exp() - b.exp()).log(): a numeracal safe implementation""" + """ + Compute log(exp(a) - exp(b)) in a numerically stable way. + + This function calculates log(exp(a) - exp(b)) using a numerically stable + implementation to avoid overflow and underflow issues that can occur when + dealing with very large or very small exponents. + + Args: + a (torch.Tensor): The first input tensor. + b (torch.Tensor): The second input tensor. Should have the same shape as 'a'. + + Returns: + torch.Tensor: The result of log(exp(a) - exp(b)), computed element-wise. + + Note: + This function handles cases where 'a' or 'b' contain infinite values. + + Examples: + >>> import torch + >>> a = torch.tensor([1.0, 2.0, 3.0]) + >>> b = torch.tensor([0.5, 1.5, 2.5]) + >>> result = log_substraction_exp(a, b) + >>> print(result) + tensor([0.9401, 1.5501, 2.5501]) + """ ans = torch.ones_like(a) * float("-inf") # avoid -inf in input diff --git a/espnet2/asr/ctc.py b/espnet2/asr/ctc.py index 13621a47cf7..1c519f86264 100644 --- a/espnet2/asr/ctc.py +++ b/espnet2/asr/ctc.py @@ -7,16 +7,47 @@ class CTC(torch.nn.Module): - """CTC module. + """ + CTC (Connectionist Temporal Classification) module. + + This class implements various CTC loss functions for sequence-to-sequence models, + particularly useful in speech recognition tasks. + + Attributes: + ctc_lo (torch.nn.Linear): Linear layer for CTC output. + ctc_loss (callable): CTC loss function based on the specified type. + dropout_rate (float): Dropout rate applied to the input. + ctc_type (str): Type of CTC loss to use ('builtin', 'builtin2', 'gtnctc', or 'brctc'). + reduce (bool): Whether to reduce the CTC loss into a scalar. Args: - odim: dimension of outputs - encoder_output_size: number of encoder projection units - dropout_rate: dropout rate (0.0 ~ 1.0) - ctc_type: builtin or gtnctc - reduce: reduce the CTC loss into a scalar - ignore_nan_grad: Same as zero_infinity (keeping for backward compatiblity) - zero_infinity: Whether to zero infinite losses and the associated gradients. + odim (int): Dimension of outputs. + encoder_output_size (int): Number of encoder projection units. + dropout_rate (float, optional): Dropout rate (0.0 ~ 1.0). Defaults to 0.0. + ctc_type (str, optional): Type of CTC loss. Defaults to "builtin". + reduce (bool, optional): Whether to reduce the CTC loss. Defaults to True. + ignore_nan_grad (bool, optional): Ignore NaN gradients (deprecated, use zero_infinity). + zero_infinity (bool, optional): Zero infinite losses and associated gradients. Defaults to True. + brctc_risk_strategy (str, optional): Risk strategy for Bayes Risk CTC. Defaults to "exp". + brctc_group_strategy (str, optional): Group strategy for Bayes Risk CTC. Defaults to "end". + brctc_risk_factor (float, optional): Risk factor for Bayes Risk CTC. Defaults to 0.0. + + Raises: + ValueError: If ctc_type is not one of "builtin", "gtnctc", or "brctc". + ImportError: If K2 is not installed when using Bayes Risk CTC. + + Note: + The class supports different CTC implementations, including the built-in PyTorch CTC, + GTN-based CTC, and Bayes Risk CTC. The choice of CTC type affects the behavior and + performance of the loss calculation. + + Example: + >>> ctc = CTC(odim=1000, encoder_output_size=256, ctc_type="builtin") + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) + >>> hlens = torch.full((32,), 100) # (batch_size,) + >>> ys_pad = torch.randint(0, 1000, (32, 50)) # (batch_size, max_label_length) + >>> ys_lens = torch.randint(10, 50, (32,)) # (batch_size,) + >>> loss = ctc(hs_pad, hlens, ys_pad, ys_lens) """ @typechecked @@ -73,6 +104,41 @@ def __init__( self.reduce = reduce def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: + """ + Calculate the CTC loss based on the specified CTC type. + + This method computes the CTC loss using the predefined CTC loss function, + which varies depending on the CTC type specified during initialization. + + Args: + th_pred (torch.Tensor): Predicted probabilities or logits. + Shape: (batch_size, max_time, num_classes) + th_target (torch.Tensor): Target labels. + Shape: (sum(target_lengths)) + th_ilen (torch.Tensor): Input lengths. + Shape: (batch_size,) + th_olen (torch.Tensor): Output lengths. + Shape: (batch_size,) + + Returns: + torch.Tensor: Computed CTC loss. + + Raises: + NotImplementedError: If an unsupported CTC type is specified. + + Note: + - For 'builtin' and 'brctc' types, the input is expected to be log probabilities. + - For 'builtin2', NaN gradients are handled differently based on the 'ignore_nan_grad' flag. + - For 'gtnctc', the input is converted to log probabilities within the method. + + Example: + >>> ctc = CTC(odim=10, encoder_output_size=20, ctc_type="builtin") + >>> pred = torch.randn(2, 5, 10) # (batch_size, max_time, num_classes) + >>> target = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) # (sum(target_lengths)) + >>> input_length = torch.tensor([5, 5]) # (batch_size,) + >>> target_length = torch.tensor([3, 5]) # (batch_size,) + >>> loss = ctc.loss_fn(pred, target, input_length, target_length) + """ if self.ctc_type == "builtin" or self.ctc_type == "brctc": th_pred = th_pred.log_softmax(2) loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) @@ -151,13 +217,38 @@ def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: raise NotImplementedError def forward(self, hs_pad, hlens, ys_pad, ys_lens): - """Calculate CTC loss. + """ + Calculate CTC loss for the input sequences. + + This method applies the CTC loss calculation to the input hidden state sequences. + It first applies a linear transformation and dropout to the input, then computes + the CTC loss based on the specified CTC type. Args: - hs_pad: batch of padded hidden state sequences (B, Tmax, D) - hlens: batch of lengths of hidden state sequences (B) - ys_pad: batch of padded character id sequence tensor (B, Lmax) - ys_lens: batch of lengths of character sequence (B) + hs_pad (torch.Tensor): Batch of padded hidden state sequences. + Shape: (batch_size, max_time, hidden_size) + hlens (torch.Tensor): Batch of lengths of hidden state sequences. + Shape: (batch_size,) + ys_pad (torch.Tensor): Batch of padded character id sequence tensor. + Shape: (batch_size, max_label_length) + ys_lens (torch.Tensor): Batch of lengths of character sequences. + Shape: (batch_size,) + + Returns: + torch.Tensor: Computed CTC loss. + + Note: + - The method handles different CTC types ('brctc', 'gtnctc', and others) differently. + - For 'gtnctc', the target sequences are converted to a list format. + - For other types, the target sequences are flattened into a 1D tensor. + + Example: + >>> ctc = CTC(odim=1000, encoder_output_size=256) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) + >>> hlens = torch.full((32,), 100) # (batch_size,) + >>> ys_pad = torch.randint(0, 1000, (32, 50)) # (batch_size, max_label_length) + >>> ys_lens = torch.randint(10, 50, (32,)) # (batch_size,) + >>> loss = ctc(hs_pad, hlens, ys_pad, ys_lens) """ # hs_pad: (B, L, NProj) -> ys_hat: (B, L, Nvocab) ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate)) @@ -184,31 +275,74 @@ def forward(self, hs_pad, hlens, ys_pad, ys_lens): return loss def softmax(self, hs_pad): - """softmax of frame activations + """ + Apply softmax to frame activations. + + This method applies a linear transformation followed by softmax to the input + hidden state sequences, typically used for obtaining output probabilities. Args: - Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. + Shape: (batch_size, max_time, hidden_size) + Returns: - torch.Tensor: softmax applied 3d tensor (B, Tmax, odim) + torch.Tensor: Softmax applied 3D tensor. + Shape: (batch_size, max_time, output_dim) + + Example: + >>> ctc = CTC(odim=1000, encoder_output_size=256) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) + >>> softmax_output = ctc.softmax(hs_pad) + >>> softmax_output.shape + torch.Size([32, 100, 1000]) """ return F.softmax(self.ctc_lo(hs_pad), dim=2) def log_softmax(self, hs_pad): - """log_softmax of frame activations + """ + Apply log softmax to frame activations. + + This method applies a linear transformation followed by log softmax to the input + hidden state sequences, typically used for obtaining log probabilities. Args: - Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. + Shape: (batch_size, max_time, hidden_size) + Returns: - torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim) + torch.Tensor: Log softmax applied 3D tensor. + Shape: (batch_size, max_time, output_dim) + + Example: + >>> ctc = CTC(odim=1000, encoder_output_size=256) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) + >>> log_softmax_output = ctc.log_softmax(hs_pad) + >>> log_softmax_output.shape + torch.Size([32, 100, 1000]) """ return F.log_softmax(self.ctc_lo(hs_pad), dim=2) def argmax(self, hs_pad): - """argmax of frame activations + """ + Apply argmax to frame activations. + + This method applies a linear transformation followed by argmax to the input + hidden state sequences, typically used for obtaining the most likely class + for each time step. Args: - torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. + Shape: (batch_size, max_time, hidden_size) + Returns: - torch.Tensor: argmax applied 2d tensor (B, Tmax) + torch.Tensor: Argmax applied 2D tensor. + Shape: (batch_size, max_time) + + Example: + >>> ctc = CTC(odim=1000, encoder_output_size=256) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) + >>> argmax_output = ctc.argmax(hs_pad) + >>> argmax_output.shape + torch.Size([32, 100]) """ return torch.argmax(self.ctc_lo(hs_pad), dim=2) diff --git a/espnet2/asr/decoder/abs_decoder.py b/espnet2/asr/decoder/abs_decoder.py index e46d1c24fcb..6497c94024b 100644 --- a/espnet2/asr/decoder/abs_decoder.py +++ b/espnet2/asr/decoder/abs_decoder.py @@ -7,6 +7,30 @@ class AbsDecoder(torch.nn.Module, ScorerInterface, ABC): + """ + Abstract base class for decoders in speech recognition models. + + This class defines the interface for decoders used in the ESPnet framework. + It inherits from torch.nn.Module for neural network functionality, + ScorerInterface for scoring methods, and ABC for abstract base class behavior. + + Attributes: + None + + Note: + Subclasses must implement the `forward` method. + + Examples: + >>> class MyDecoder(AbsDecoder): + ... def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens): + ... # Implement decoder logic here + ... pass + ... + ... def score(self, ys, state, x): + ... # Implement scoring logic here + ... pass + """ + @abstractmethod def forward( self, @@ -15,4 +39,39 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Forward pass of the decoder. + + This abstract method defines the forward pass for the decoder. It takes + encoded features and target sequences as input and produces decoded output. + + Args: + hs_pad (torch.Tensor): Padded hidden state sequences from the encoder. + Shape: (batch, time, hidden_dim) + hlens (torch.Tensor): Lengths of hidden state sequences. + Shape: (batch,) + ys_in_pad (torch.Tensor): Padded input token sequences for teacher forcing. + Shape: (batch, output_length) + ys_in_lens (torch.Tensor): Lengths of input token sequences. + Shape: (batch,) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - Decoded output sequences. Shape: (batch, output_length, vocab_size) + - Attention weights or None. Shape: (batch, output_length, input_length) + + Raises: + NotImplementedError: This method must be implemented by subclasses. + + Note: + This method should be implemented by all subclasses of AbsDecoder. + + Examples: + >>> class MyDecoder(AbsDecoder): + ... def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens): + ... # Decoder implementation + ... decoded_output = ... # Shape: (batch, output_length, vocab_size) + ... attention_weights = ... # Shape: (batch, output_length, input_length) + ... return decoded_output, attention_weights + """ raise NotImplementedError diff --git a/espnet2/asr/decoder/hugging_face_transformers_decoder.py b/espnet2/asr/decoder/hugging_face_transformers_decoder.py index 1af31b3679d..464af8217a0 100644 --- a/espnet2/asr/decoder/hugging_face_transformers_decoder.py +++ b/espnet2/asr/decoder/hugging_face_transformers_decoder.py @@ -27,11 +27,39 @@ class HuggingFaceTransformersDecoder(AbsDecoder, BatchScorerInterface): - """Hugging Face Transformers Decoder. + """ + A decoder class that utilizes Hugging Face Transformers models. + + This class implements a decoder interface using pre-trained models from the + Hugging Face Transformers library. It supports both causal language models + and sequence-to-sequence models. + + Attributes: + causal_lm (bool): Whether the model is a causal language model. + decoder (torch.nn.Module): The main decoder component from the Hugging Face model. + decoder_word_embeddings (torch.nn.Embedding): Word embeddings of the decoder. + decoder_pad_token_id (int): Padding token ID for the decoder. + tokenizer_padding_side (str): Padding side used by the tokenizer ('left' or 'right'). + prefix (torch.Tensor): Embedded prefix tokens. + postfix (torch.Tensor): Embedded postfix tokens. + lm_head (torch.nn.Module): Language model head for output projection. + model_name_or_path (str): Name or path of the Hugging Face model. + linear_in (torch.nn.Module): Linear layer to match encoder output size to decoder input size. Args: - encoder_output_size: dimension of encoder attention - model_name_or_path: Hugging Face Transformers model name + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimension of the encoder output. + model_name_or_path (str): Name or path of the Hugging Face Transformers model. + causal_lm (bool, optional): Whether to use a causal language model. Defaults to False. + prefix (str, optional): Prefix to add to the input. Defaults to "". + postfix (str, optional): Postfix to add to the input. Defaults to "". + + Raises: + ImportError: If the 'transformers' library is not installed. + + Note: + This class requires the 'transformers' library to be installed. + It adapts Hugging Face models for use with the ESPnet framework. """ @typechecked @@ -117,19 +145,26 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the decoder. + + This method processes the encoder output and generates decoder output scores. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: input tensor (batch, maxlen_out, #mels) - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: + hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of the encoded sequences in the batch. + ys_in_pad (torch.Tensor): Input tensor of shape (batch, maxlen_out, #mels). + ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch. - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - x (torch.Tensor): Decoded token scores before softmax, + of shape (batch, maxlen_out, token). + - ys_in_lens (torch.Tensor): Lengths of the input sequences. + + Note: + The method handles both causal language models and sequence-to-sequence models. + For causal language models, it adds prefix and postfix to the input. """ enc_out = self.linear_in(hs_pad) @@ -184,7 +219,18 @@ def forward( return x, ys_in_lens def reload_pretrained_parameters(self): - self.decoder.load_state_dict(self.decoder_pretrained_params) + """ + Reloads the pretrained parameters for the decoder and language model head. + + This method restores the original pretrained weights of the Hugging Face + Transformers model. It resets both the decoder and the language model head + to their initial pretrained states. + + Note: + This method is useful for resetting the model to its pretrained state, + which can be beneficial in certain training or fine-tuning scenarios. + It logs a message upon successful reloading of the parameters. + """ if self.lm_head_pretrained_params is not None: self.lm_head.load_state_dict(self.lm_head_pretrained_params) @@ -192,6 +238,33 @@ def reload_pretrained_parameters(self): logging.info("Pretrained Transformers model parameters reloaded!") def add_prefix_postfix(self, enc_out, hlens, ys_in_pad, ys_in_lens): + """ + Adds prefix and postfix to the encoder output for causal language models. + + This method prepares the input for causal language models by adding + prefix and postfix embeddings to the encoder output. It also handles + padding and creates the appropriate attention mask. + + Args: + enc_out (torch.Tensor): Encoder output tensor. + hlens (torch.Tensor): Lengths of the encoder outputs in the batch. + ys_in_pad (torch.Tensor): Input tensor (batch, maxlen_out, #mels). + ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch. + + Returns: + Tuple[Dict[str, torch.Tensor], torch.Tensor]: + - args (Dict[str, torch.Tensor]): A dictionary containing: + - 'inputs_embeds': Tensor with added prefix, postfix, and padding. + - 'attention_mask': Attention mask for the inputs. + - 'return_dict': Boolean set to True. + - no_loss_lengths (torch.Tensor): Lengths of the sequences excluding + the parts that should not contribute to the loss calculation. + + Note: + This method is specifically designed for use with causal language models + in the HuggingFaceTransformersDecoder class. It handles different padding + sides based on the tokenizer configuration. + """ args = {} hlens_max = (hlens + ys_in_lens).max() @@ -236,6 +309,28 @@ def add_prefix_postfix(self, enc_out, hlens, ys_in_pad, ys_in_lens): return args, no_loss_lengths def score(self, ys, state, x, speech=None): + """ + Computes the score for the next token given the current state and input. + + This method generates the log probability scores for the next token in the sequence + using the Hugging Face model's generation process. + + Args: + ys (torch.Tensor): Current token sequence. + state (Any): The current state (not used in this implementation). + x (torch.Tensor): The encoder output. + speech (torch.Tensor, optional): Speech input (not used in this implementation). + + Returns: + Tuple[torch.Tensor, None]: + - next_token_scores (torch.Tensor): Log probability scores for the next token. + - None: Placeholder for the updated state (not used in this implementation). + + Note: + This method is part of the BatchScorerInterface and is used in beam search + decoding. It prepares the input for the Hugging Face model, runs the forward + pass, and returns the log probabilities for the next token. + """ model_kwargs = { "encoder_outputs": ModelOutput( last_hidden_state=self.linear_in(x).unsqueeze(0) @@ -264,6 +359,30 @@ def batch_score( xs: torch.Tensor, speech: torch.Tensor = None, ) -> Tuple[torch.Tensor, List[Any]]: + """ + Computes scores for the next tokens in a batch of sequences. + + This method generates log probability scores for the next tokens in multiple + sequences simultaneously using the Hugging Face model's generation process. + + Args: + ys (torch.Tensor): Batch of current token sequences. + states (List[Any]): List of current states (not used in this implementation). + xs (torch.Tensor): Batch of encoder outputs. + speech (torch.Tensor, optional): Batch of speech inputs (not used in this implementation). + + Returns: + Tuple[torch.Tensor, None]: + - next_token_scores (torch.Tensor): Log probability scores for the next tokens + in the batch. Shape: (batch_size, vocab_size). + - None: Placeholder for the updated states (not used in this implementation). + + Note: + This method is part of the BatchScorerInterface and is used for efficient + batch processing in beam search decoding. It prepares the input for the + Hugging Face model, runs the forward pass, and returns the log probabilities + for the next tokens in all sequences of the batch. + """ # import pdb;pdb.set_trace() model_kwargs = { "encoder_outputs": ModelOutput(last_hidden_state=self.linear_in(xs)), @@ -285,6 +404,33 @@ def batch_score( def get_hugging_face_model_network(model): + """ + Retrieves the network component from a Hugging Face model. + + This function attempts to extract the main network component from various + Hugging Face model architectures. It checks for common attribute names + used in different model implementations. + + Args: + model: A Hugging Face model object. + + Returns: + The network component of the model. + + Raises: + Exception: If the network attribute cannot be found in the model. + + Examples: + >>> from transformers import AutoModelForCausalLM + >>> model = AutoModelForCausalLM.from_pretrained("gpt2") + >>> network = get_hugging_face_model_network(model) + >>> print(type(network).__name__) + 'GPT2Model' + + Note: + This function supports models with 'transformer', 'gpt_neox', or 'model' + attributes. If none of these attributes are found, an exception is raised. + """ if hasattr(model, "transformer"): network = model.transformer elif hasattr(model, "gpt_neox"): @@ -298,6 +444,33 @@ def get_hugging_face_model_network(model): def get_hugging_face_model_lm_head(model): + """ + Retrieves the language model head component from a Hugging Face model. + + This function attempts to extract the language model head component from + various Hugging Face model architectures. It checks for common attribute + names used in different model implementations. + + Args: + model: A Hugging Face model object. + + Returns: + The language model head component of the model. + + Raises: + Exception: If the LM head attribute cannot be found in the model. + + Examples: + >>> from transformers import AutoModelForCausalLM + >>> model = AutoModelForCausalLM.from_pretrained("gpt2") + >>> lm_head = get_hugging_face_model_lm_head(model) + >>> print(type(lm_head).__name__) + 'Linear' + + Note: + This function supports models with 'lm_head' or 'embed_out' attributes. + If neither of these attributes are found, an exception is raised. + """ if hasattr(model, "lm_head"): lm_head = model.lm_head elif hasattr(model, "embed_out"): diff --git a/espnet2/asr/decoder/mlm_decoder.py b/espnet2/asr/decoder/mlm_decoder.py index a787185de11..344f39301a8 100644 --- a/espnet2/asr/decoder/mlm_decoder.py +++ b/espnet2/asr/decoder/mlm_decoder.py @@ -20,6 +20,50 @@ class MLMDecoder(AbsDecoder): + """ + Masked Language Model (MLM) Decoder for sequence-to-sequence models. + + This class implements a decoder for Masked Language Modeling tasks, typically + used in transformer-based architectures. It processes encoded input sequences + and generates output sequences with the ability to handle masked tokens. + + Attributes: + embed (torch.nn.Sequential): Input embedding layer. + normalize_before (bool): Whether to apply layer normalization before each decoder block. + after_norm (LayerNorm): Layer normalization applied after all decoder blocks if normalize_before is True. + output_layer (torch.nn.Linear or None): Linear layer for final output projection. + decoders (torch.nn.ModuleList): List of decoder layers. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in position-wise feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer, either "embed" or "linear". Defaults to "embed". + use_output_layer (bool, optional): Whether to use an output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to apply layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + + Note: + This decoder adds an extra token to the vocabulary size to account for the mask token. + + Example: + >>> encoder_output_size = 256 + >>> vocab_size = 1000 + >>> decoder = MLMDecoder(vocab_size, encoder_output_size) + >>> hs_pad = torch.randn(32, 50, encoder_output_size) # (batch, max_len, feat) + >>> hlens = torch.full((32,), 50, dtype=torch.long) + >>> ys_in_pad = torch.randint(0, vocab_size, (32, 30)) # (batch, max_len) + >>> ys_in_lens = torch.full((32,), 30, dtype=torch.long) + >>> decoded, olens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) + """ + @typechecked def __init__( self, @@ -90,21 +134,43 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the MLM Decoder. + + This method processes the input sequences through the decoder layers, + applying self-attention, source attention, and feed-forward operations. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) + hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). + ys_in_pad (torch.Tensor): Input token ids, int64 tensor of shape (batch, maxlen_out) + if input_layer is "embed", or input tensor of shape (batch, maxlen_out, #mels) + for other input layer types. + ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). + Returns: - (tuple): tuple containing: - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + tuple: A tuple containing: + - x (torch.Tensor): Decoded token scores before softmax, shape (batch, maxlen_out, token) + if use_output_layer is True. If use_output_layer is False, returns the + last decoder layer's output. + - olens (torch.Tensor): Output sequence lengths, shape (batch,). + + Raises: + ValueError: If the input shapes are inconsistent with the expected dimensions. + + Example: + >>> decoder = MLMDecoder(1000, 256) + >>> hs_pad = torch.randn(32, 50, 256) + >>> hlens = torch.full((32,), 50, dtype=torch.long) + >>> ys_in_pad = torch.randint(0, 1000, (32, 30)) + >>> ys_in_lens = torch.full((32,), 30, dtype=torch.long) + >>> decoded, olens = decoder.forward(hs_pad, hlens, ys_in_pad, ys_in_lens) + >>> print(decoded.shape, olens.shape) + torch.Size([32, 30, 1001]) torch.Size([32]) + + Note: + The method handles masking for both the target and memory sequences to ensure + proper attention mechanisms during decoding. """ tgt = ys_in_pad # tgt_mask: (B, 1, L) diff --git a/espnet2/asr/decoder/rnn_decoder.py b/espnet2/asr/decoder/rnn_decoder.py index 634108357a2..61738f8f212 100644 --- a/espnet2/asr/decoder/rnn_decoder.py +++ b/espnet2/asr/decoder/rnn_decoder.py @@ -30,6 +30,50 @@ def build_attention_list( han_conv_filts: int = 100, han_win: int = 5, ): + """ + Build a list of attention modules based on the specified parameters. + + This function creates and returns a list of attention modules for use in + speech recognition models. It supports both single and multi-encoder + configurations, as well as hierarchical attention networks (HAN). + + Args: + eprojs (int): Number of encoder projection units. + dunits (int): Number of decoder units. + atype (str, optional): Attention type. Defaults to "location". + num_att (int, optional): Number of attention modules. Defaults to 1. + num_encs (int, optional): Number of encoders. Defaults to 1. + aheads (int, optional): Number of attention heads. Defaults to 4. + adim (int, optional): Attention dimension. Defaults to 320. + awin (int, optional): Attention window size. Defaults to 5. + aconv_chans (int, optional): Number of attention convolution channels. Defaults to 10. + aconv_filts (int, optional): Number of attention convolution filters. Defaults to 100. + han_mode (bool, optional): Whether to use hierarchical attention network. Defaults to False. + han_type (str, optional): Type of hierarchical attention. Defaults to None. + han_heads (int, optional): Number of HAN attention heads. Defaults to 4. + han_dim (int, optional): HAN attention dimension. Defaults to 320. + han_conv_chans (int, optional): Number of HAN convolution channels. Defaults to -1. + han_conv_filts (int, optional): Number of HAN convolution filters. Defaults to 100. + han_win (int, optional): HAN window size. Defaults to 5. + + Returns: + torch.nn.ModuleList: A list of attention modules. + + Raises: + ValueError: If the number of encoders is less than one. + + Note: + The function behavior changes based on the number of encoders and whether + hierarchical attention network mode is enabled. + + Examples: + >>> att_list = build_attention_list(256, 320, num_encs=2, han_mode=True) + >>> print(len(att_list)) + 1 + >>> att_list = build_attention_list(256, 320, num_encs=2, han_mode=False) + >>> print(len(att_list)) + 2 + """ att_list = torch.nn.ModuleList() if num_encs == 1: for i in range(num_att): @@ -80,6 +124,54 @@ def build_attention_list( class RNNDecoder(AbsDecoder): + """ + RNN-based decoder for sequence-to-sequence models. + + This class implements a recurrent neural network (RNN) decoder, which can be + used in various sequence-to-sequence tasks such as speech recognition or + machine translation. It supports both LSTM and GRU cell types, multiple + layers, and various attention mechanisms. + + The decoder uses an embedding layer, followed by multiple RNN layers with + dropout, and an output layer. It also incorporates attention mechanisms to + focus on different parts of the input sequence during decoding. + + Attributes: + embed (torch.nn.Embedding): Embedding layer for input tokens. + decoder (torch.nn.ModuleList): List of RNN cells (LSTM or GRU). + dropout_dec (torch.nn.ModuleList): List of dropout layers for RNN outputs. + output (torch.nn.Linear): Output layer for vocabulary distribution. + att_list (torch.nn.ModuleList): List of attention modules. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + rnn_type (str, optional): Type of RNN cell to use ('lstm' or 'gru'). Defaults to "lstm". + num_layers (int, optional): Number of RNN layers. Defaults to 1. + hidden_size (int, optional): Hidden size of the RNN. Defaults to 320. + sampling_probability (float, optional): Probability of sampling from previous output. Defaults to 0.0. + dropout (float, optional): Dropout probability. Defaults to 0.0. + context_residual (bool, optional): Whether to use context residual connection. Defaults to False. + replace_sos (bool, optional): Whether to replace token. Defaults to False. + num_encs (int, optional): Number of encoders. Defaults to 1. + att_conf (dict, optional): Configuration for attention modules. Defaults to get_default_kwargs(build_attention_list). + + Raises: + ValueError: If an unsupported RNN type is specified. + + Note: + This decoder supports both single and multi-encoder configurations, as well as + speaker parallel attention (SPA) for multi-speaker scenarios. + + Example: + >>> decoder = RNNDecoder(vocab_size=1000, encoder_output_size=256, num_layers=2, hidden_size=512) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, encoder_output_size) + >>> hlens = torch.full((32,), 100) # (batch_size,) + >>> ys_in_pad = torch.randint(0, 1000, (32, 20)) # (batch_size, max_output_length) + >>> ys_in_lens = torch.full((32,), 20) # (batch_size,) + >>> decoder_output, output_lengths = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) + """ + @typechecked def __init__( self, @@ -150,9 +242,65 @@ def __init__( ) def zero_state(self, hs_pad): + """ + Initialize a zero state for the decoder. + + This method creates a tensor of zeros with the same batch size as the input + and the decoder's hidden size. It's typically used to initialize the hidden + state of the RNN at the start of the decoding process. + + Args: + hs_pad (torch.Tensor): The padded hidden state tensor from the encoder. + It is used to determine the batch size for the zero state. + + Returns: + torch.Tensor: A tensor of zeros with shape (batch_size, hidden_size), + where batch_size is inferred from hs_pad and hidden_size is the + decoder's hidden size (self.dunits). + + Example: + >>> decoder = RNNDecoder(...) + >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, encoder_output_size) + >>> initial_state = decoder.zero_state(hs_pad) + >>> print(initial_state.shape) + torch.Size([32, 320]) # Assuming self.dunits = 320 + """ return hs_pad.new_zeros(hs_pad.size(0), self.dunits) def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): + """ + Perform a forward pass through the RNN layers. + + This method processes the input through all layers of the RNN (LSTM or GRU), + applying dropout between vertical connections. + + Args: + ey (torch.Tensor): Input tensor, typically the concatenation of the + embedded previous output and the context vector. + z_list (list): List to store output hidden states for each layer. + c_list (list): List to store cell states for each layer (used for LSTM only). + z_prev (list): List of previous hidden states for each layer. + c_prev (list): List of previous cell states for each layer (used for LSTM only). + + Returns: + tuple: A tuple containing: + - z_list (list): Updated list of output hidden states for each layer. + - c_list (list): Updated list of cell states for each layer (for LSTM only). + + Note: + - For LSTM, both hidden state (z) and cell state (c) are updated. + - For GRU, only the hidden state (z) is updated. + - Dropout is applied to the output of each layer except the last one. + + Example: + >>> decoder = RNNDecoder(...) + >>> ey = torch.randn(32, 512) # (batch_size, embed_size + context_size) + >>> z_list = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] + >>> c_list = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] + >>> z_prev = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] + >>> c_prev = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] + >>> z_list, c_list = decoder.rnn_forward(ey, z_list, c_list, z_prev, c_prev) + """ if self.dtype == "lstm": z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) for i in range(1, self.dlayers): @@ -169,6 +317,47 @@ def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): return z_list, c_list def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): + """ + Perform a forward pass of the RNN decoder. + + This method processes the encoder outputs and generates decoder outputs + for the entire sequence. + + Args: + hs_pad (torch.Tensor or List[torch.Tensor]): Padded hidden states from encoder(s). + For single encoder: tensor of shape (batch, time, hidden_size). + For multiple encoders: list of such tensors. + hlens (torch.Tensor or List[torch.Tensor]): Lengths of encoder hidden states. + For single encoder: tensor of shape (batch,). + For multiple encoders: list of such tensors. + ys_in_pad (torch.Tensor): Padded input label sequences. + Shape: (batch, sequence_length). + ys_in_lens (torch.Tensor): Lengths of input label sequences. + Shape: (batch,). + strm_idx (int, optional): Stream index for multi-speaker attention. + Defaults to 0. + + Returns: + tuple: A tuple containing: + - z_all (torch.Tensor): Output sequence scores. + Shape: (batch, sequence_length, vocab_size). + - ys_in_lens (torch.Tensor): Lengths of input sequences. + + Note: + - This method supports both single and multiple encoder scenarios. + - It implements teacher forcing with optional scheduled sampling. + - For multiple encoders, hierarchical attention is used. + + Example: + >>> decoder = RNNDecoder(...) + >>> hs_pad = torch.randn(32, 100, 256) # (batch, time, hidden_size) + >>> hlens = torch.full((32,), 100) + >>> ys_in_pad = torch.randint(0, 1000, (32, 20)) # (batch, sequence_length) + >>> ys_in_lens = torch.full((32,), 20) + >>> z_all, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) + >>> print(z_all.shape) + torch.Size([32, 20, 1000]) # (batch, sequence_length, vocab_size) + """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: @@ -253,6 +442,41 @@ def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): return z_all, ys_in_lens def init_state(self, x): + """ + Initialize the decoder state for inference. + + This method sets up the initial state of the decoder, including hidden states, + cell states (for LSTM), and attention weights. It's typically used at the start + of the decoding process during inference. + + Args: + x (torch.Tensor or List[torch.Tensor]): The encoder output(s). + For single encoder: tensor of shape (batch, time, hidden_size). + For multiple encoders: list of such tensors. + + Returns: + dict: A dictionary containing the initial decoder state with keys: + - c_prev (list): Initial cell states for each layer (for LSTM). + - z_prev (list): Initial hidden states for each layer. + - a_prev (None or list): Initial attention weights. + - workspace (tuple): Additional workspace information including: + - att_idx (int): Index of the current attention module. + - z_list (list): List of initial hidden states. + - c_list (list): List of initial cell states (for LSTM). + + Note: + - The method handles both single and multiple encoder scenarios. + - For multiple encoders, it initializes states for all encoders and the + hierarchical attention network (HAN). + - The attention modules are reset to clear any pre-computed values. + + Example: + >>> decoder = RNNDecoder(...) + >>> x = torch.randn(1, 100, 256) # (batch, time, hidden_size) + >>> initial_state = decoder.init_state(x) + >>> print(initial_state.keys()) + dict_keys(['c_prev', 'z_prev', 'a_prev', 'workspace']) + """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: @@ -282,6 +506,44 @@ def init_state(self, x): ) def score(self, yseq, state, x): + """ + Calculate the log probability score for the next token. + + This method computes the score for the next token in the sequence given the + current state and encoder outputs. It's typically used in beam search decoding. + + Args: + yseq (torch.Tensor): Current output sequence. + Shape: (sequence_length,). + state (dict): Current decoder state containing: + - c_prev (list): Previous cell states for each layer (for LSTM). + - z_prev (list): Previous hidden states for each layer. + - a_prev (None or list): Previous attention weights. + - workspace (tuple): Additional workspace information. + x (torch.Tensor or List[torch.Tensor]): The encoder output(s). + For single encoder: tensor of shape (batch, time, hidden_size). + For multiple encoders: list of such tensors. + + Returns: + tuple: A tuple containing: + - logp (torch.Tensor): Log probability scores for the next token. + Shape: (vocab_size,). + - new_state (dict): Updated decoder state. + + Note: + - This method supports both single and multiple encoder scenarios. + - For multiple encoders, it uses hierarchical attention. + - The context vector is concatenated with the embedded input for scoring. + + Example: + >>> decoder = RNNDecoder(...) + >>> yseq = torch.tensor([1, 2, 3]) # Current sequence + >>> x = torch.randn(1, 100, 256) # (batch, time, hidden_size) + >>> state = decoder.init_state(x) + >>> logp, new_state = decoder.score(yseq, state, x) + >>> print(logp.shape) + torch.Size([1000]) # Assuming vocab_size = 1000 + """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: diff --git a/espnet2/asr/decoder/s4_decoder.py b/espnet2/asr/decoder/s4_decoder.py index 30828bcb23d..cfa02217267 100644 --- a/espnet2/asr/decoder/s4_decoder.py +++ b/espnet2/asr/decoder/s4_decoder.py @@ -12,25 +12,43 @@ class S4Decoder(AbsDecoder, BatchScorerInterface): - """S4 decoder module. + """ + S4 decoder module for sequence-to-sequence models. + + This class implements a decoder based on the S4 (Structured State Space Sequence) model. + It can be used in various sequence-to-sequence tasks, such as automatic speech recognition. Args: - vocab_size: output dim - encoder_output_size: dimension of hidden vector - input_layer: input layer type - dropinp: input dropout - dropout: dropout parameter applied on every residual and every layer - prenorm: pre-norm vs. post-norm - n_layers: number of layers - transposed: transpose inputs so each layer receives (batch, dim, length) - tie_dropout: tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d - n_repeat: each layer is repeated n times per stage before applying pooling - layer: layer config, must be specified - residual: residual config - norm: normalization config (e.g. layer vs batch) - pool: config for pooling layer per stage - track_norms: log norms of each layer output - drop_path: drop rate for stochastic depth + vocab_size (int): Size of the vocabulary (output dimension). + encoder_output_size (int): Dimension of the encoder's hidden vector. + input_layer (str, optional): Type of input layer. Defaults to "embed". + dropinp (float, optional): Input dropout rate. Defaults to 0.0. + dropout (float, optional): Dropout rate applied to every residual and layer. Defaults to 0.25. + prenorm (bool, optional): If True, use pre-normalization; otherwise, post-normalization. Defaults to True. + n_layers (int, optional): Number of layers in the decoder. Defaults to 16. + transposed (bool, optional): If True, transpose inputs so each layer receives (batch, dim, length). Defaults to False. + tie_dropout (bool, optional): If True, tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d. Defaults to False. + n_repeat (int, optional): Number of times each layer is repeated per stage before applying pooling. Defaults to 1. + layer (dict, optional): Layer configuration. Must be specified. + residual (dict, optional): Residual connection configuration. + norm (dict, optional): Normalization configuration (e.g., layer vs batch). + pool (dict, optional): Configuration for pooling layer per stage. + track_norms (bool, optional): If True, log norms of each layer output. Defaults to True. + drop_path (float, optional): Drop rate for stochastic depth. Defaults to 0.0. + + Attributes: + d_model (int): Dimension of the model (same as encoder_output_size). + sos (int): Start-of-sequence token ID. + eos (int): End-of-sequence token ID. + odim (int): Output dimension (same as vocab_size). + dropout (float): Dropout rate. + embed (torch.nn.Embedding): Embedding layer for input tokens. + dropout_emb (torch.nn.Dropout): Dropout layer for embeddings. + decoder (SequenceModel): Main S4 decoder model. + output (torch.nn.Linear): Output linear layer. + + Note: + This decoder implements the BatchScorerInterface, allowing for efficient batch scoring of hypotheses. """ @typechecked @@ -87,7 +105,27 @@ def __init__( self.output = torch.nn.Linear(self.d_model, vocab_size) def init_state(self, x: torch.Tensor): - """Initialize state.""" + """ + Initialize the decoder state. + + This method initializes the state of the S4 decoder. It creates a default state + for the decoder based on the input tensor's device. + + Args: + x (torch.Tensor): An input tensor used to determine the device for state initialization. + + Returns: + Any: The initialized state of the decoder. + + Note: + The state is initialized with a batch size of 1, regardless of the input tensor's batch size. + This method is typically called before starting the decoding process. + + Example: + >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) + >>> x = torch.randn(1, 10, 512) # (batch_size, sequence_length, feature_dim) + >>> initial_state = decoder.init_state(x) + """ return self.decoder.default_state(1, device=x.device) def forward( @@ -98,22 +136,39 @@ def forward( ys_in_lens: torch.Tensor, state=None, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the S4 decoder. + + This method performs the forward pass of the S4 decoder, processing the encoder output + and the input token sequence to generate decoded token scores. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) + hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of the encoded sequences in the batch, shape (batch,). + ys_in_pad (torch.Tensor): Input token ids, int64 tensor of shape (batch, maxlen_out). + If input_layer is "embed", this contains token ids. + Otherwise, it contains input tensors of shape (batch, maxlen_out, #mels). + ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch, shape (batch,). + state (Optional[Any]): Initial state for the decoder. Defaults to None. + Returns: - (tuple): tuple containing: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - decoded (torch.Tensor): Decoded token scores before softmax, + shape (batch, maxlen_out, vocab_size). + - ys_in_lens (torch.Tensor): Lengths of the input sequences, shape (batch,). - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + Note: + This method applies the embedding layer, processes the sequence through the S4 decoder, + and applies the output layer to generate token scores. + + Example: + >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) + >>> hs_pad = torch.randn(2, 100, 512) # (batch_size, max_encoder_length, encoder_dim) + >>> hlens = torch.tensor([100, 80]) + >>> ys_in_pad = torch.randint(0, 1000, (2, 20)) # (batch_size, max_decoder_length) + >>> ys_in_lens = torch.tensor([20, 15]) + >>> decoded, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) + >>> print(decoded.shape) # (2, 20, 1000) """ memory = hs_pad memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( @@ -133,24 +188,61 @@ def forward( return decoded, ys_in_lens def score(self, ys, state, x): + """ + Score a sequence of tokens. + + This method is not implemented for the S4Decoder. + + Args: + ys: The sequence of tokens to be scored. + state: The current state of the decoder. + x: The input features. + + Raises: + NotImplementedError: This method is not implemented for S4Decoder. + + Note: + This method is part of the BatchScorerInterface, but it is not implemented + for the S4Decoder. Use the `batch_score` method instead for efficient + batch scoring of hypotheses. + """ raise NotImplementedError def batch_score( self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. + """ + Score new token batches efficiently. + + This method computes scores for the next token given a batch of prefix tokens + and their corresponding states. It is part of the BatchScorerInterface and is + used for efficient batch decoding. Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). + ys (torch.Tensor): Prefix tokens of shape (n_batch, ylen). + Contains int64 token IDs. + states (List[Any]): List of scorer states for prefix tokens. + Each state corresponds to a sequence in the batch. + xs (torch.Tensor): The encoder features that generate ys. + Shape (n_batch, xlen, n_feat). Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. + Tuple[torch.Tensor, List[Any]]: A tuple containing: + - logp (torch.Tensor): Log probabilities for the next token. + Shape (n_batch, n_vocab). + - states_list (List[Any]): Updated states for each sequence in the batch. + + Note: + This method is designed for use in beam search decoding, where multiple + hypotheses are scored simultaneously. + Example: + >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) + >>> ys = torch.randint(0, 1000, (5, 10)) # (n_batch, ylen) + >>> states = [decoder.init_state(torch.randn(1, 512)) for _ in range(5)] + >>> xs = torch.randn(5, 100, 512) # (n_batch, xlen, n_feat) + >>> logp, new_states = decoder.batch_score(ys, states, xs) + >>> print(logp.shape) # (5, 1000) """ # merge states n_batch = len(ys) diff --git a/espnet2/asr/decoder/transducer_decoder.py b/espnet2/asr/decoder/transducer_decoder.py index 857a7d09215..0d198ecc2e5 100644 --- a/espnet2/asr/decoder/transducer_decoder.py +++ b/espnet2/asr/decoder/transducer_decoder.py @@ -10,17 +10,48 @@ class TransducerDecoder(AbsDecoder): - """(RNN-)Transducer decoder module. + """ + (RNN-)Transducer decoder module. + + This class implements a decoder for (RNN-)Transducer models, which can be used + in automatic speech recognition (ASR) systems. It supports both LSTM and GRU + as the underlying recurrent neural network types. + + Attributes: + embed: Embedding layer for input tokens. + dropout_embed: Dropout layer applied to the embedding output. + decoder: List of RNN layers (LSTM or GRU). + dropout_dec: List of dropout layers applied after each RNN layer. + dlayers: Number of decoder layers. + dunits: Number of hidden units in each decoder layer. + dtype: Type of RNN used ('lstm' or 'gru'). + odim: Output dimension (vocabulary size). + ignore_id: ID to ignore in the input (default: -1). + blank_id: ID representing the blank/pad token. + device: Device on which the model is allocated. Args: - vocab_size: Output dimension. - layers_type: (RNN-)Decoder layers type. - num_layers: Number of decoder layers. - hidden_size: Number of decoder units per layer. - dropout: Dropout rate for decoder layers. - dropout_embed: Dropout rate for embedding layer. - embed_pad: Embed/Blank symbol ID. - + vocab_size: Size of the vocabulary (output dimension). + rnn_type: Type of RNN to use ('lstm' or 'gru'). Defaults to 'lstm'. + num_layers: Number of decoder layers. Defaults to 1. + hidden_size: Number of hidden units in each decoder layer. Defaults to 320. + dropout: Dropout rate for decoder layers. Defaults to 0.0. + dropout_embed: Dropout rate for the embedding layer. Defaults to 0.0. + embed_pad: ID of the padding token in the embedding layer. Defaults to 0. + + Raises: + ValueError: If an unsupported RNN type is specified. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> labels = torch.randint(0, 1000, (32, 10)) # Batch of 32, sequence length 10 + >>> output = decoder(labels) + >>> print(output.shape) + torch.Size([32, 10, 320]) # (batch_size, sequence_length, hidden_size) + + Note: + This implementation follows the Google Python Style Guide and PEP 8 standards. + The decoder can be used as part of a larger (RNN-)Transducer model for ASR tasks. """ @typechecked @@ -66,25 +97,56 @@ def __init__( self.device = next(self.parameters()).device def set_device(self, device: torch.device): - """Set GPU device to use. + """ + Set GPU device to use. - Args: - device: Device ID. + This method sets the device (GPU) on which the decoder will operate. + Args: + device (torch.device): The PyTorch device object representing the GPU + to be used for computations. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000) + >>> device = torch.device("cuda:0") + >>> decoder.set_device(device) + + Note: + This method should be called before performing any computations if you want + to explicitly set the GPU device. If not called, the decoder will use the + device of its parameters by default. """ self.device = device def init_state( self, batch_size: int ) -> Tuple[torch.Tensor, Optional[torch.tensor]]: - """Initialize decoder states. + """ + Initialize decoder states. + + This method initializes the hidden states of the decoder for a given batch size. + It creates zero tensors for both LSTM (hidden state and cell state) and GRU (hidden state only). Args: - batch_size: Batch size. + batch_size (int): The batch size for which to initialize the states. Returns: - : Initial decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - + Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - h_n (torch.Tensor): The initial hidden state tensor of shape (N, B, D_dec), + where N is the number of layers, B is the batch size, and D_dec is the + decoder's hidden size. + - c_n (Optional[torch.Tensor]): The initial cell state tensor for LSTM, + with the same shape as h_n. For GRU, this will be None. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> batch_size = 32 + >>> h_n, c_n = decoder.init_state(batch_size) + >>> print(h_n.shape, c_n.shape) + torch.Size([2, 32, 320]) torch.Size([2, 32, 320]) + + Note: + The initialized states are created on the same device as the decoder. """ h_n = torch.zeros( self.dlayers, @@ -110,16 +172,38 @@ def rnn_forward( sequence: torch.Tensor, state: Tuple[torch.Tensor, Optional[torch.Tensor]], ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """Encode source label sequences. + """ + Encode source label sequences through the RNN layers. + + This method passes the input sequence through all RNN layers of the decoder, + applying dropout after each layer. Args: - sequence: RNN input sequences. (B, D_emb) - state: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + sequence (torch.Tensor): RNN input sequences of shape (B, D_emb), + where B is the batch size and D_emb is the embedding dimension. + state (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. + For LSTM, it's a tuple of (hidden_state, cell_state), each of shape (N, B, D_dec). + For GRU, it's a tuple of (hidden_state, None). + N is the number of layers, B is the batch size, and D_dec is the decoder's hidden size. Returns: - sequence: RNN output sequences. (B, D_dec) - (h_next, c_next): Decoder hidden states. (N, B, D_dec), (N, B, D_dec)) - + Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: + - sequence (torch.Tensor): RNN output sequences of shape (B, D_dec). + - (h_next, c_next) (Tuple[torch.Tensor, Optional[torch.Tensor]]): + Updated decoder hidden states. For LSTM, both h_next and c_next + have shape (N, B, D_dec). For GRU, c_next is None. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> batch_size, seq_len, hidden_size = 32, 10, 320 + >>> input_seq = torch.randn(batch_size, hidden_size) + >>> initial_state = decoder.init_state(batch_size) + >>> output_seq, (h_next, c_next) = decoder.rnn_forward(input_seq, initial_state) + >>> print(output_seq.shape, h_next.shape, c_next.shape) + torch.Size([32, 320]) torch.Size([2, 32, 320]) torch.Size([2, 32, 320]) + + Note: + This method handles both LSTM and GRU architectures based on the decoder's configuration. """ h_prev, c_prev = state h_next, c_next = self.init_state(sequence.size(0)) @@ -145,14 +229,33 @@ def rnn_forward( return sequence, (h_next, c_next) def forward(self, labels: torch.Tensor) -> torch.Tensor: - """Encode source label sequences. + """ + Encode source label sequences. + + This method processes input label sequences through the decoder's embedding layer + and RNN layers to produce decoder output sequences. Args: - labels: Label ID sequences. (B, L) + labels (torch.Tensor): Label ID sequences of shape (B, L), where B is the + batch size and L is the sequence length. Returns: - dec_out: Decoder output sequences. (B, T, U, D_dec) - + torch.Tensor: Decoder output sequences of shape (B, T, U, D_dec), where + T is the number of time steps, U is the number of label tokens, and + D_dec is the decoder's hidden size. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> batch_size, seq_length = 32, 10 + >>> labels = torch.randint(0, 1000, (batch_size, seq_length)) + >>> output = decoder(labels) + >>> print(output.shape) + torch.Size([32, 10, 320]) + + Note: + This method applies dropout to the embedded input before passing it through + the RNN layers. The output shape might differ from the example if the decoder + is configured differently. """ init_state = self.init_state(labels.size(0)) dec_embed = self.dropout_embed(self.embed(labels)) @@ -164,17 +267,35 @@ def forward(self, labels: torch.Tensor) -> torch.Tensor: def score( self, hyp: Hypothesis, cache: Dict[str, Any] ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]], torch.Tensor]: - """One-step forward hypothesis. + """ + Perform one-step forward computation for a single hypothesis. + + This method computes the decoder output for the next step given a current hypothesis. + It uses a cache to store and retrieve previously computed results for efficiency. Args: - hyp: Hypothesis. - cache: Pairs of (dec_out, state) for each label sequence. (key) + hyp (Hypothesis): The current hypothesis containing the label sequence and decoder state. + cache (Dict[str, Any]): A dictionary storing pairs of (dec_out, state) for each label sequence. Returns: - dec_out: Decoder output sequence. (1, D_dec) - new_state: Decoder hidden states. ((N, 1, D_dec), (N, 1, D_dec)) - label: Label ID for LM. (1,) - + Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]], torch.Tensor]: + - dec_out (torch.Tensor): Decoder output sequence of shape (1, D_dec). + - new_state (Tuple[torch.Tensor, Optional[torch.Tensor]]): Updated decoder hidden states. + For LSTM, it's (hidden_state, cell_state), each of shape (N, 1, D_dec). + For GRU, it's (hidden_state, None). + - label (torch.Tensor): Label ID for language model, shape (1,). + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000) + >>> hyp = Hypothesis(...) # Create a hypothesis + >>> cache = {} + >>> dec_out, new_state, label = decoder.score(hyp, cache) + >>> print(dec_out.shape, label.shape) + torch.Size([1, 320]) torch.Size([1]) + + Note: + This method is typically used in beam search decoding for Transducer models. + The cache helps avoid redundant computations for previously seen label sequences. """ label = torch.full((1, 1), hyp.yseq[-1], dtype=torch.long, device=self.device) @@ -197,19 +318,40 @@ def batch_score( cache: Dict[str, Any], use_lm: bool, ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: - """One-step forward hypotheses. + """ + Perform one-step forward computation for a batch of hypotheses. + + This method computes the decoder output for the next step given a batch of current hypotheses. + It uses a cache to store and retrieve previously computed results for efficiency. Args: - hyps: Hypotheses. - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - cache: Pairs of (dec_out, dec_states) for each label sequences. (keys) - use_lm: Whether to compute label ID sequences for LM. + hyps (Union[List[Hypothesis], List[ExtendedHypothesis]]): A list of current hypotheses. + dec_states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. + For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). + For GRU, it's (hidden_state, None). + cache (Dict[str, Any]): A dictionary storing pairs of (dec_out, dec_states) for each label sequence. + use_lm (bool): Whether to compute label ID sequences for language model integration. Returns: - dec_out: Decoder output sequences. (B, D_dec) - dec_states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - lm_labels: Label ID sequences for LM. (B,) - + Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + - dec_out (torch.Tensor): Decoder output sequences of shape (B, D_dec). + - dec_states (Tuple[torch.Tensor, torch.Tensor]): Updated decoder hidden states. + For LSTM, both elements have shape (N, B, D_dec). For GRU, the second element is None. + - lm_labels (torch.Tensor): Label ID sequences for language model, shape (B,). + Returns None if use_lm is False. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000) + >>> hyps = [Hypothesis(...) for _ in range(5)] # Batch of 5 hypotheses + >>> dec_states = decoder.init_state(5) + >>> cache = {} + >>> dec_out, new_states, lm_labels = decoder.batch_score(hyps, dec_states, cache, use_lm=True) + >>> print(dec_out.shape, lm_labels.shape) + torch.Size([5, 320]) torch.Size([5, 1]) + + Note: + This method is typically used in batch beam search decoding for Transducer models. + It's more efficient than scoring hypotheses individually, especially for larger batch sizes. """ final_batch = len(hyps) @@ -258,16 +400,35 @@ def batch_score( def select_state( self, states: Tuple[torch.Tensor, Optional[torch.Tensor]], idx: int ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Get specified ID state from decoder hidden states. + """ + Extract a specific state from the decoder hidden states. + + This method selects and returns the hidden state for a given index from the batch + of decoder hidden states. Args: - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - idx: State ID to extract. + states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. + For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). + For GRU, it's (hidden_state, None). + N is the number of layers, B is the batch size, and D_dec is the decoder's hidden size. + idx (int): The index of the state to extract. Returns: - : Decoder hidden state for given ID. - ((N, 1, D_dec), (N, 1, D_dec)) - + Tuple[torch.Tensor, Optional[torch.Tensor]]: The selected decoder hidden state. + - For LSTM: (hidden_state, cell_state), each of shape (N, 1, D_dec). + - For GRU: (hidden_state, None), where hidden_state has shape (N, 1, D_dec). + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> batch_size = 32 + >>> states = decoder.init_state(batch_size) + >>> selected_state = decoder.select_state(states, 5) + >>> print(selected_state[0].shape, selected_state[1].shape) + torch.Size([2, 1, 320]) torch.Size([2, 1, 320]) + + Note: + This method is useful when you need to extract the state for a specific + hypothesis from a batch of states, typically during beam search decoding. """ return ( states[0][:, idx : idx + 1, :], @@ -280,15 +441,40 @@ def create_batch_states( new_states: List[Tuple[torch.Tensor, Optional[torch.Tensor]]], check_list: Optional[List] = None, ) -> List[Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """Create decoder hidden states. + """ + Create batch decoder hidden states from individual states. + + This method combines individual decoder states into a batch, which is useful + for parallel processing of multiple hypotheses. Args: - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - new_states: Decoder hidden states. [N x ((1, D_dec), (1, D_dec))] + states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Initial decoder hidden states. + For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). + For GRU, it's (hidden_state, None). + new_states (List[Tuple[torch.Tensor, Optional[torch.Tensor]]]): List of individual + decoder hidden states to be combined. + Each element is of shape (N, 1, D_dec) for LSTM hidden and cell states, + or (N, 1, D_dec) and None for GRU. + check_list (Optional[List]): Not used in the current implementation. + Kept for potential future use or compatibility. Returns: - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) - + Tuple[torch.Tensor, Optional[torch.Tensor]]: Combined batch of decoder hidden states. + - For LSTM: (hidden_state, cell_state), each of shape (N, B, D_dec). + - For GRU: (hidden_state, None), where hidden_state has shape (N, B, D_dec). + N is the number of layers, B is the new batch size (length of new_states), + and D_dec is the decoder's hidden size. + + Example: + >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) + >>> individual_states = [decoder.init_state(1) for _ in range(5)] + >>> batch_states = decoder.create_batch_states(decoder.init_state(5), individual_states) + >>> print(batch_states[0].shape, batch_states[1].shape) + torch.Size([2, 5, 320]) torch.Size([2, 5, 320]) + + Note: + This method is particularly useful in beam search decoding when combining + states from different hypotheses into a single batch for efficient processing. """ return ( torch.cat([s[0] for s in new_states], dim=1), diff --git a/espnet2/asr/decoder/transformer_decoder.py b/espnet2/asr/decoder/transformer_decoder.py index 0386a4c7e48..9ccc8860dbb 100644 --- a/espnet2/asr/decoder/transformer_decoder.py +++ b/espnet2/asr/decoder/transformer_decoder.py @@ -26,25 +26,32 @@ class BaseTransformerDecoder(AbsDecoder, BatchScorerInterface): - """Base class of Transfomer decoder module. + """ + Base class for Transformer decoder modules. + + This abstract base class provides the foundation for implementing various + Transformer decoder architectures. It defines the common structure and + methods that all Transformer decoder variants should implement. + + Attributes: + embed (torch.nn.Sequential): The embedding layer for input tokens. + decoders (torch.nn.ModuleList): List of decoder layers (to be implemented by subclasses). + after_norm (LayerNorm): Layer normalization applied after the decoder stack. + output_layer (torch.nn.Linear): Linear layer for final output projection. Args: - vocab_size: output dim - encoder_output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - self_attention_dropout_rate: dropout rate for attention - input_layer: input layer type - use_output_layer: whether to use output layer - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + dropout_rate (float): Dropout rate. + positional_dropout_rate (float): Dropout rate for positional encoding. + input_layer (str): Type of input layer ('embed' or 'linear'). + use_output_layer (bool): Whether to use an output layer. + pos_enc_class: Positional encoding class to use. + normalize_before (bool): Whether to apply layer normalization before each block. + + Note: + Subclasses should implement the specific decoder architecture by + defining the `decoders` attribute. """ @typechecked @@ -99,25 +106,33 @@ def forward( return_hs: bool = False, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the decoder. + + This method processes the encoder output and generates decoded sequences. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - return_hs: (bool) whether to return the last hidden output - before output layer - return_all_hs: (bool) whether to return all the hidden intermediates - Returns: - (tuple): tuple containing: + hs_pad (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). + ys_in_pad (torch.Tensor): Input token ids or features, shape (batch, maxlen_out). + If input_layer is "embed", it contains token ids. + Otherwise, it contains input features. + ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). + return_hs (bool, optional): Whether to return the last hidden state. Defaults to False. + return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + Returns: + tuple: A tuple containing: + - x (torch.Tensor): Decoded token scores before softmax, + shape (batch, maxlen_out, vocab_size) if use_output_layer is True. + - olens (torch.Tensor): Output lengths, shape (batch,). + - hidden (torch.Tensor, optional): Last hidden state if return_hs is True. + - intermediate_outs (List[torch.Tensor], optional): All intermediate hidden states + if return_all_hs is True. + + Note: + The behavior of this method can be customized using the return_hs and + return_all_hs flags to obtain additional hidden state information. """ tgt = ys_in_pad # tgt_mask: (B, 1, L) @@ -170,21 +185,29 @@ def forward_one_step( cache: List[torch.Tensor] = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. + """ + Perform one step of the decoder forward pass. + + This method is typically used for incremental decoding, processing one token at a time. Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoded memory mask (batch, 1, maxlen_in) - cache: cached output list of (batch, max_time_out-1, size) - return_hs: dec hidden state corresponding to ys, - used for searchable hidden ints + tgt (torch.Tensor): Input token ids, shape (batch, maxlen_out). + tgt_mask (torch.Tensor): Input token mask, shape (batch, maxlen_out). + dtype=torch.uint8 in PyTorch 1.2-, dtype=torch.bool in PyTorch 1.2+ (including 1.2). + memory (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). + memory_mask (torch.Tensor, optional): Encoded memory mask, shape (batch, 1, maxlen_in). + cache (List[torch.Tensor], optional): Cached output list of shape (batch, max_time_out-1, size). + return_hs (bool, optional): Whether to return the hidden state. Defaults to False. + Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) + tuple: A tuple containing: + - y (torch.Tensor): Output tensor of shape (batch, maxlen_out, token) + if use_output_layer is True, else the last hidden state. + - new_cache (List[torch.Tensor]): Updated cache for each decoder layer. + - hidden (torch.Tensor, optional): Hidden state if return_hs is True. + + Note: + This method is crucial for efficient autoregressive decoding in inference time. """ x = self.embed(tgt) if cache is None: @@ -210,7 +233,28 @@ def forward_one_step( return y, new_cache def score(self, ys, state, x, return_hs=False): - """Score.""" + """ + Score a sequence of tokens. + + This method computes the log probability score for a given sequence of tokens. + + Args: + ys (torch.Tensor): Sequence of tokens to score, shape (sequence_length,). + state (List[Any]): Previous decoder state. + x (torch.Tensor): Encoder output, shape (1, encoder_output_size). + return_hs (bool, optional): Whether to return the hidden state. Defaults to False. + + Returns: + tuple: A tuple containing: + - logp (torch.Tensor): Log probability scores for the input sequence, + shape (sequence_length, vocab_size). + - state (List[Any]): Updated decoder state. + - hs (torch.Tensor, optional): Hidden state if return_hs is True. + + Note: + This method is typically used in beam search and other decoding algorithms + to evaluate and rank candidate sequences. + """ ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) if return_hs: (logp, hs), state = self.forward_one_step( @@ -238,20 +282,27 @@ def batch_score( xs: torch.Tensor, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. + """ + Score a batch of token sequences. - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). + This method computes the log probability scores for a batch of token sequences. + Args: + ys (torch.Tensor): Batch of token sequences, shape (batch_size, sequence_length). + states (List[Any]): List of scorer states for prefix tokens. + xs (torch.Tensor): Batch of encoder outputs, shape (batch_size, max_length, encoder_output_size). + return_hs (bool, optional): Whether to return the hidden states. Defaults to False. Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - + tuple: A tuple containing: + - logp (torch.Tensor): Batch of log probability scores for the next token, + shape (batch_size, vocab_size). + - state_list (List[List[Any]]): Updated list of scorer states for each sequence in the batch. + - hs (torch.Tensor, optional): Hidden states if return_hs is True. + + Note: + This method is optimized for batch processing, which is more efficient than + scoring sequences individually, especially during beam search or batch decoding. """ # merge states n_batch = len(ys) @@ -284,6 +335,40 @@ def batch_score( class TransformerDecoder(BaseTransformerDecoder): + """ + Transformer decoder module. + + This class implements the standard Transformer decoder architecture as described + in "Attention Is All You Need" (Vaswani et al., 2017). It consists of multiple + stacked self-attention and encoder-decoder attention layers, followed by a + feed-forward network. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + layer_drop_rate (float, optional): Layer dropout rate. Defaults to 0.0. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers. + + Note: + This implementation allows for easy modification of various hyperparameters + and architectural choices, making it suitable for a wide range of sequence + generation tasks. + """ + @typechecked def __init__( self, @@ -335,6 +420,41 @@ def __init__( class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder): + """ + Lightweight Convolution Transformer decoder module. + + This class implements a Transformer decoder that replaces the self-attention + mechanism with lightweight convolution, as described in "Pay Less Attention + with Lightweight and Dynamic Convolutions" (Wu et al., 2019). It combines + the benefits of convolutional neural networks and self-attention. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + conv_wshare (int, optional): Weight sharing factor for convolution. Defaults to 4. + conv_kernel_length (Sequence[int], optional): Kernel size for each convolution layer. Defaults to (11, 11, 11, 11, 11, 11). + conv_usebias (bool, optional): Whether to use bias in convolution layers. Defaults to False. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers with lightweight convolution. + + Note: + This decoder variant can be more efficient than standard self-attention + for certain tasks, especially those involving long sequences. + """ + @typechecked def __init__( self, @@ -397,6 +517,41 @@ def __init__( class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder): + """ + Lightweight 2D Convolution Transformer decoder module. + + This class implements a Transformer decoder that uses 2D lightweight convolutions + instead of self-attention. It extends the concept introduced in "Pay Less Attention + with Lightweight and Dynamic Convolutions" (Wu et al., 2019) to 2D convolutions, + potentially capturing more complex patterns in the input. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + conv_wshare (int, optional): Weight sharing factor for 2D convolution. Defaults to 4. + conv_kernel_length (Sequence[int], optional): Kernel size for each 2D convolution layer. Defaults to (11, 11, 11, 11, 11, 11). + conv_usebias (bool, optional): Whether to use bias in 2D convolution layers. Defaults to False. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers with 2D lightweight convolution. + + Note: + This decoder variant may be particularly effective for tasks where the input has + a 2D structure or where capturing 2D patterns is beneficial. + """ + @typechecked def __init__( self, @@ -459,6 +614,43 @@ def __init__( class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder): + """ + Dynamic Convolution Transformer decoder module. + + This class implements a Transformer decoder that replaces the self-attention + mechanism with dynamic convolution, as introduced in "Pay Less Attention with + Lightweight and Dynamic Convolutions" (Wu et al., 2019). Dynamic convolution + adapts its weights based on the input, allowing for more flexible and + context-dependent processing. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + conv_wshare (int, optional): Weight sharing factor for dynamic convolution. Defaults to 4. + conv_kernel_length (Sequence[int], optional): Kernel size for each dynamic convolution layer. Defaults to (11, 11, 11, 11, 11, 11). + conv_usebias (bool, optional): Whether to use bias in dynamic convolution layers. Defaults to False. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers with dynamic convolution. + + Note: + Dynamic convolution can be more effective than standard self-attention or + lightweight convolution for certain tasks, especially those requiring + adaptive processing of input sequences. + """ + @typechecked def __init__( self, @@ -521,6 +713,43 @@ def __init__( class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder): + """ + Dynamic 2D Convolution Transformer decoder module. + + This class implements a Transformer decoder that uses 2D dynamic convolutions + instead of self-attention. It extends the concept of dynamic convolutions + introduced in "Pay Less Attention with Lightweight and Dynamic Convolutions" + (Wu et al., 2019) to 2D, allowing for more complex and adaptive processing + of input sequences with potential 2D structure. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + conv_wshare (int, optional): Weight sharing factor for 2D dynamic convolution. Defaults to 4. + conv_kernel_length (Sequence[int], optional): Kernel size for each 2D dynamic convolution layer. Defaults to (11, 11, 11, 11, 11, 11). + conv_usebias (bool, optional): Whether to use bias in 2D dynamic convolution layers. Defaults to False. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers with 2D dynamic convolution. + + Note: + This decoder variant may be particularly effective for tasks where the input + has a 2D structure or where capturing adaptive 2D patterns is beneficial. + It combines the flexibility of dynamic convolutions with 2D processing capabilities. + """ + @typechecked def __init__( self, @@ -583,6 +812,40 @@ def __init__( class TransformerMDDecoder(BaseTransformerDecoder): + """ + Transformer Multi-Decoder (MD) module. + + This class implements a Transformer decoder with an additional attention mechanism + for speech input, making it suitable for multi-modal tasks such as speech translation + or speech recognition with auxiliary text input. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Dimensionality of the encoder output. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of decoder layers. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. + src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. + input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". + use_output_layer (bool, optional): Whether to use output layer. Defaults to True. + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + use_speech_attn (bool, optional): Whether to use additional attention for speech input. Defaults to True. + + Attributes: + decoders (torch.nn.ModuleList): List of decoder layers with additional speech attention mechanism. + use_speech_attn (bool): Indicates whether speech attention is used. + + Note: + This decoder is designed for tasks that involve both text and speech inputs, + allowing for more effective integration of multi-modal information. The additional + speech attention mechanism can be toggled on or off based on the task requirements. + """ + @typechecked def __init__( self, @@ -650,24 +913,34 @@ def forward( speech_lens: torch.Tensor = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the TransformerMDDecoder. + + This method processes the encoder output, speech input (if applicable), and generates + decoded sequences, optionally returning intermediate hidden states. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - return_hs: dec hidden state corresponding to ys, - used for searchable hidden ints - Returns: - (tuple): tuple containing: + hs_pad (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). + ys_in_pad (torch.Tensor): Input token ids or features, shape (batch, maxlen_out). + If input_layer is "embed", it contains token ids. + Otherwise, it contains input features. + ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). + speech (torch.Tensor, optional): Speech input tensor, shape (batch, speech_maxlen, speech_feat). + speech_lens (torch.Tensor, optional): Lengths of speech input sequences, shape (batch,). + return_hs (bool, optional): Whether to return the last hidden state. Defaults to False. - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + Returns: + tuple: A tuple containing: + - x (torch.Tensor): Decoded token scores before softmax, + shape (batch, maxlen_out, vocab_size) if use_output_layer is True. + - olens (torch.Tensor): Output lengths, shape (batch,). + - hs_asr (torch.Tensor, optional): Last hidden state if return_hs is True. + + Note: + This method supports multi-modal decoding by incorporating both text and speech + inputs when available. The speech attention mechanism is used only if + use_speech_attn is True and speech input is provided. """ tgt = ys_in_pad # tgt_mask: (B, 1, L) @@ -724,23 +997,34 @@ def forward_one_step( cache: List[torch.Tensor] = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. + """ + Perform one step of the decoder forward pass. + + This method is designed for incremental decoding, processing one token at a time + and optionally incorporating speech input. Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoded memory mask (batch, 1, maxlen_in) - speech: encoded speech, float32 (batch, maxlen_in, feat) - speech_mask: encoded memory mask (batch, 1, maxlen_in) - cache: cached output list of (batch, max_time_out-1, size) - return_hs: dec hidden state corresponding to ys, - used for searchable hidden ints + tgt (torch.Tensor): Input token ids, shape (batch, maxlen_out). + tgt_mask (torch.Tensor): Input token mask, shape (batch, maxlen_out). + dtype=torch.uint8 in PyTorch 1.2-, dtype=torch.bool in PyTorch 1.2+ (including 1.2). + memory (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). + memory_mask (torch.Tensor, optional): Encoded memory mask, shape (batch, 1, maxlen_in). + speech (torch.Tensor, optional): Speech input tensor, shape (batch, speech_maxlen, speech_feat). + speech_mask (torch.Tensor, optional): Speech input mask, shape (batch, 1, speech_maxlen). + cache (List[torch.Tensor], optional): Cached output list of shape (batch, max_time_out-1, size). + return_hs (bool, optional): Whether to return the hidden state. Defaults to False. + Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) + tuple: A tuple containing: + - y (torch.Tensor): Output tensor of shape (batch, maxlen_out, token) + if use_output_layer is True, else the last hidden state. + - h_asr (torch.Tensor, optional): Hidden state if return_hs is True. + - new_cache (List[torch.Tensor]): Updated cache for each decoder layer. + + Note: + This method supports multi-modal decoding by incorporating both text and speech + inputs when available. The speech attention mechanism is used only if + use_speech_attn is True and speech input is provided. """ x = self.embed(tgt) if cache is None: @@ -779,7 +1063,30 @@ def forward_one_step( return y, new_cache def score(self, ys, state, x, speech=None): - """Score.""" + """ + Score a sequence of tokens. + + This method computes the log probability score for a given sequence of tokens, + optionally incorporating speech input. + + Args: + ys (torch.Tensor): Sequence of tokens to score, shape (sequence_length,). + state (List[Any]): Previous decoder state. + x (torch.Tensor): Encoder output, shape (1, encoder_output_size). + speech (torch.Tensor, optional): Speech input tensor, shape (1, speech_maxlen, speech_feat). + + Returns: + tuple: A tuple containing: + - logp (torch.Tensor): Log probability scores for the input sequence, + shape (sequence_length, vocab_size). + - state (List[Any]): Updated decoder state. + + Note: + This method supports multi-modal scoring by incorporating both text and speech + inputs when available. The speech attention mechanism is used only if + use_speech_attn is True and speech input is provided. It is typically used + in beam search and other decoding algorithms to evaluate candidate sequences. + """ ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) logp, state = self.forward_one_step( ys.unsqueeze(0), @@ -797,19 +1104,30 @@ def batch_score( xs: torch.Tensor, speech: torch.Tensor = None, ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. + """ + Score a batch of token sequences. + + This method computes the log probability scores for a batch of token sequences, + optionally incorporating speech input for multi-modal scoring. Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). + ys (torch.Tensor): Batch of token sequences, shape (batch_size, sequence_length). + states (List[Any]): List of scorer states for prefix tokens. + xs (torch.Tensor): Batch of encoder outputs, shape (batch_size, max_length, encoder_output_size). + speech (torch.Tensor, optional): Batch of speech inputs, shape (batch_size, speech_maxlen, speech_feat). Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - + tuple: A tuple containing: + - logp (torch.Tensor): Batch of log probability scores for the next token, + shape (batch_size, vocab_size). + - state_list (List[List[Any]]): Updated list of scorer states for each sequence in the batch. + + Note: + This method is optimized for batch processing, which is more efficient than + scoring sequences individually. It supports multi-modal scoring by incorporating + both text and speech inputs when available. The speech attention mechanism is used + only if use_speech_attn is True and speech input is provided. This method is + particularly useful for beam search or batch decoding in multi-modal tasks. """ # merge states n_batch = len(ys) diff --git a/espnet2/asr/decoder/whisper_decoder.py b/espnet2/asr/decoder/whisper_decoder.py index b0106bd1bf5..17c157252ee 100644 --- a/espnet2/asr/decoder/whisper_decoder.py +++ b/espnet2/asr/decoder/whisper_decoder.py @@ -9,6 +9,34 @@ class ExpandedTokenEmbedding(torch.nn.Module): + """ + A custom embedding layer that expands an existing embedding with additional tokens. + + This class extends the functionality of a given embedding layer by adding + more token embeddings while preserving the original embeddings. The new + embeddings are initialized with normal distribution based on the statistics + of the original embedding weights. + + Attributes: + ori_emb (torch.nn.Embedding): The original embedding layer. + add_emb (torch.nn.Embedding): The additional embedding layer for new tokens. + num_embeddings (int): Total number of embeddings (original + additional). + + Args: + ori_emebedding (torch.nn.Embedding): The original embedding layer to be expanded. + additional_size (int): Number of additional token embeddings to add. + + Note: + The forward method is overridden to use the combined weights of both + original and additional embeddings. + + Example: + >>> original_embedding = torch.nn.Embedding(1000, 300) + >>> expanded_embedding = ExpandedTokenEmbedding(original_embedding, 500) + >>> input_tensor = torch.LongTensor([0, 1500, 999]) + >>> output = expanded_embedding(input_tensor) + """ + def __init__(self, ori_emebedding, additional_size): super().__init__() self.ori_emb = ori_emebedding @@ -24,9 +52,42 @@ def __init__(self, ori_emebedding, additional_size): @property def weight(self): + """ + Combined weight tensor of the original and additional embeddings. + + Returns: + torch.Tensor: A tensor containing the concatenated weights of the original + embedding (ori_emb) and the additional embedding (add_emb) along dimension 0. + + Note: + This property is used to provide a unified view of the entire embedding + weight, including both original and additional token embeddings. + """ return torch.cat([self.ori_emb.weight, self.add_emb.weight], dim=0) def forward(self, input): + """ + Performs a forward pass through the expanded embedding layer. + + This method applies the embedding operation using the combined weights + of the original and additional embeddings. It preserves the properties + of the original embedding layer, such as padding_idx, max_norm, etc. + + Args: + input (torch.Tensor): Input tensor containing token indices. + + Returns: + torch.Tensor: The embedded representation of the input tokens. + + Note: + This method overrides the default forward pass of torch.nn.Embedding + to use the combined weights while maintaining other embedding properties. + + Example: + >>> expanded_embedding = ExpandedTokenEmbedding(original_embedding, 500) + >>> input_tensor = torch.LongTensor([0, 1500, 999]) + >>> output = expanded_embedding(input_tensor) + """ return torch.nn.functional.embedding( input, self.weight, @@ -39,9 +100,40 @@ def forward(self, input): class OpenAIWhisperDecoder(AbsDecoder, BatchScorerInterface): - """Transformer-based Speech-to-Text Decoder from OpenAI's Whisper Model: - - URL: https://github.com/openai/whisper + """ + A decoder class based on OpenAI's Whisper model for speech-to-text tasks. + + This class implements a Transformer-based decoder that utilizes the architecture + from OpenAI's Whisper model. It can be used for various speech recognition and + transcription tasks. + + Attributes: + decoders (whisper.model.Decoder): The Whisper model's decoder. + dropout (torch.nn.Dropout): Dropout layer for regularization. + load_origin_token_embedding (bool): Flag to load original token embeddings. + + Args: + vocab_size (int): Size of the vocabulary. + encoder_output_size (int): Size of the encoder's output. + dropout_rate (float, optional): Dropout rate. Defaults to 0.0. + whisper_model (str, optional): Whisper model size. Defaults to "small". + download_dir (str, optional): Directory to download the Whisper model. + load_origin_token_embedding (bool, optional): Whether to load original + token embeddings when expanding vocabulary. Defaults to False. + + Raises: + Exception: If the Whisper package is not properly installed. + + Note: + This class inherits from AbsDecoder and BatchScorerInterface, providing + compatibility with the ESPnet2 framework. + + Example: + >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) + >>> encoder_output = torch.randn(1, 100, 512) + >>> decoder_input = torch.LongTensor([[1, 2, 3, 4, 5]]) + >>> decoder_output, _ = decoder(encoder_output, torch.tensor([100]), + ... decoder_input, torch.tensor([5])) """ @typechecked @@ -114,22 +206,35 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. + """ + Forward pass of the OpenAI Whisper decoder. + + This method processes the encoder output and the decoder input to generate + the output token scores. Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: + hs_pad (torch.Tensor): Encoded memory, float32 (batch, maxlen_in, feat). + hlens (torch.Tensor): Lengths of encoded sequences (batch,). + ys_in_pad (torch.Tensor): Input token ids, int64 (batch, maxlen_out). + ys_in_lens (torch.Tensor): Lengths of input sequences (batch,). - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) + Returns: + tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - x (torch.Tensor): Decoded token scores before softmax + (batch, maxlen_out, token). + - ys_in_lens (torch.Tensor): Lengths of input sequences (batch,). + + Note: + This method applies positional embedding, processes the input through + the decoder blocks, and generates the final output scores. + + Example: + >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) + >>> hs_pad = torch.randn(2, 100, 512) + >>> hlens = torch.tensor([100, 80]) + >>> ys_in_pad = torch.randint(0, 10000, (2, 20)) + >>> ys_in_lens = torch.tensor([20, 15]) + >>> output, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) """ tgt, memory = ys_in_pad, hs_pad tgt = ( @@ -160,21 +265,37 @@ def forward_one_step( *, cache: List[torch.Tensor] = None, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. + """ + Perform a single forward step in the decoder. + + This method processes one step of decoding, typically used in inference + or beam search scenarios. Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) + tgt (torch.Tensor): Input token ids, int64 (batch, maxlen_out). + tgt_mask (torch.Tensor): Input token mask, (batch, maxlen_out). + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (including 1.2) + memory (torch.Tensor): Encoded memory, float32 (batch, maxlen_in, feat). + cache (List[torch.Tensor], optional): Cached output list of + (batch, max_time_out-1, size). Defaults to None. + Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - NOTE (Shih-Lun): - cache implementation is ignored for now - for simplicity & correctness + tuple[torch.Tensor, None]: A tuple containing: + - y (torch.Tensor): Log probabilities of next tokens (batch, vocab_size). + - None: Placeholder for cache (currently not implemented). + + Note: + - The cache implementation is currently ignored for simplicity and correctness. + - This method applies positional embedding, processes through decoder blocks, + and generates log probabilities for the next tokens. + + Example: + >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) + >>> tgt = torch.LongTensor([[1, 2, 3]]) + >>> tgt_mask = torch.ones(1, 3, dtype=torch.bool) + >>> memory = torch.randn(1, 100, 512) + >>> output, _ = decoder.forward_one_step(tgt, tgt_mask, memory) """ x = ( self.decoders.token_embedding(tgt) @@ -198,7 +319,32 @@ def forward_one_step( return y, None def score(self, ys, state, x): - """Score.""" + """ + Calculate the score for the next token. + + This method computes the log probability scores for the next token given + the current state and encoder output. + + Args: + ys (torch.Tensor): Current token sequence. + state (Any): Current decoder state (unused in this implementation). + x (torch.Tensor): Encoder output. + + Returns: + tuple[torch.Tensor, None]: A tuple containing: + - logp (torch.Tensor): Log probability scores for the next token. + - None: Updated state (currently not implemented). + + Note: + This method is typically used in beam search or other decoding algorithms + to score possible next tokens. + + Example: + >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) + >>> ys = torch.LongTensor([1, 2, 3]) + >>> x = torch.randn(100, 512) + >>> logp, _ = decoder.score(ys, None, x) + """ logp, state = self.forward_one_step( ys.unsqueeze(0), torch.empty(0), x.unsqueeze(0), cache=state # dummy mask ) @@ -207,19 +353,31 @@ def score(self, ys, state, x): def batch_score( self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. + """ + Score new token batch. + + This method computes scores for the next tokens given batched inputs of + current token sequences and encoder features. Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). + ys (torch.Tensor): Prefix tokens, torch.int64 (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens (unused in this implementation). + xs (torch.Tensor): Encoder features that generate ys, (n_batch, xlen, n_feat). Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - + tuple[torch.Tensor, None]: A tuple containing: + - logp (torch.Tensor): Batchified scores for next tokens, shape (n_batch, n_vocab). + - None: Placeholder for next state list (currently not implemented). + + Note: + This method is designed for batch processing, which can be more efficient + than scoring individual sequences separately. + + Example: + >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) + >>> ys = torch.LongTensor([[1, 2, 3], [4, 5, 6]]) + >>> xs = torch.randn(2, 100, 512) + >>> logp, _ = decoder.batch_score(ys, None, xs) """ # batch decoding, dummy mask is passed logp, states = self.forward_one_step(ys, torch.empty(0), xs, cache=None) diff --git a/espnet2/asr/discrete_asr_espnet_model.py b/espnet2/asr/discrete_asr_espnet_model.py index bc0f60e2271..e3c22800910 100644 --- a/espnet2/asr/discrete_asr_espnet_model.py +++ b/espnet2/asr/discrete_asr_espnet_model.py @@ -28,7 +28,26 @@ def autocast(enabled=True): class ESPnetDiscreteASRModel(ESPnetMTModel): - """Encoder-Decoder model""" + """ + Encoder-Decoder model for Automatic Speech Recognition (ASR) with discrete units. + + This class extends the ESPnetMTModel to incorporate ASR-specific components such as + CTC (Connectionist Temporal Classification) and SpecAugment. It combines encoder-decoder + architecture with CTC and attention-based decoding for improved ASR performance. + + Attributes: + specaug (Optional[AbsSpecAug]): SpecAugment module for data augmentation. + blank_id (int): ID for the blank token in CTC. + ctc_weight (float): Weight of CTC loss in the total loss calculation. + interctc_weight (float): Weight of intermediate CTC loss. + ctc (Optional[CTC]): CTC module for ASR. + error_calculator (Optional[ASRErrorCalculator]): Calculator for ASR errors. + + Note: + This model supports both regular and intermediate CTC losses, as well as + attention-based decoding. The final loss is a weighted combination of CTC + and attention decoder losses. + """ @typechecked def __init__( @@ -109,14 +128,26 @@ def forward( src_text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss + """ + Perform forward pass of the model: Frontend + Encoder + Decoder + Loss calculation. Args: - text: (Batch, Length) - text_lengths: (Batch,) - src_text: (Batch, length) - src_text_lengths: (Batch,) - kwargs: "utt_id" is among the input. + text (torch.Tensor): Target text tensor. Shape: (Batch, Length) + text_lengths (torch.Tensor): Length of each sequence in the target text. Shape: (Batch,) + src_text (torch.Tensor): Source text tensor. Shape: (Batch, length) + src_text_lengths (torch.Tensor): Length of each sequence in the source text. Shape: (Batch,) + **kwargs: Additional keyword arguments. "utt_id" is expected among the inputs. + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: A tuple containing: + - loss (torch.Tensor): The total loss of the model. + - stats (Dict[str, torch.Tensor]): A dictionary with various statistics and metrics. + - weight (torch.Tensor): The batch size as a weight for the loss. + + Note: + This method handles the entire forward pass of the model, including CTC and + attention-based loss calculations. It also computes various metrics such as + Character Error Rate (CER) and Word Error Rate (WER) when not in training mode. """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -201,11 +232,31 @@ def forward( def encode( self, src_text: torch.Tensor, src_text_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by mt_inference.py + """ + Perform frontend processing and encoding of the input. + + This method applies the frontend processing, optional SpecAugment, pre-encoding, + main encoding, and post-encoding steps to the input. Args: - src_text: (Batch, Length, ...) - src_text_lengths: (Batch, ) + src_text (torch.Tensor): Source text tensor. Shape: (Batch, Length, ...) + src_text_lengths (torch.Tensor): Length of each sequence in the source text. Shape: (Batch,) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - encoder_out (torch.Tensor): The encoded output. Shape: (Batch, Length2, Dim2) + - encoder_out_lens (torch.Tensor): The length of each sequence in the encoded output. + + If intermediate outputs are available: + Tuple[Tuple[torch.Tensor, List[Tuple[int, torch.Tensor]]], torch.Tensor]: A tuple containing: + - (encoder_out, intermediate_outs): + - encoder_out (torch.Tensor): The final encoded output. + - intermediate_outs (List[Tuple[int, torch.Tensor]]): Intermediate encoder outputs. + - encoder_out_lens (torch.Tensor): The length of each sequence in the encoded output. + + Note: + This method is used by mt_inference.py and handles the entire encoding process, + including optional intermediate CTC outputs if the encoder supports it. """ with autocast(False): # 1. Extract feats diff --git a/espnet2/asr/encoder/abs_encoder.py b/espnet2/asr/encoder/abs_encoder.py index 22a1a103458..b57d195b3a3 100644 --- a/espnet2/asr/encoder/abs_encoder.py +++ b/espnet2/asr/encoder/abs_encoder.py @@ -5,8 +5,62 @@ class AbsEncoder(torch.nn.Module, ABC): + """ + Abstract base class for encoders in a neural network. + + This class defines the interface for encoders used in various neural network + architectures. It inherits from both torch.nn.Module and ABC (Abstract Base Class), + ensuring that it can be used as a PyTorch module while also enforcing the + implementation of abstract methods in derived classes. + + Attributes: + None + + Note: + This class should not be instantiated directly. Instead, it should be + subclassed with concrete implementations of the abstract methods. + + Examples: + ```python + class ConcreteEncoder(AbsEncoder): + def output_size(self) -> int: + return 256 + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + # Implementation details + pass + ``` + """ + @abstractmethod def output_size(self) -> int: + """ + Returns the output size of the encoder. + + This abstract method should be implemented by subclasses to specify + the size of the encoder's output. + + Returns: + int: The size of the encoder's output. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Example: + ```python + class ConcreteEncoder(AbsEncoder): + def output_size(self) -> int: + return 256 # Example output size + + encoder = ConcreteEncoder() + size = encoder.output_size() # Returns 256 + ``` + """ raise NotImplementedError @abstractmethod @@ -16,4 +70,43 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Performs the forward pass of the encoder. + + This abstract method should be implemented by subclasses to define the + forward pass of the encoder. + + Args: + xs_pad (torch.Tensor): Padded input tensor. + ilens (torch.Tensor): Input lengths tensor. + prev_states (torch.Tensor, optional): Previous encoder states. Defaults to None. + + Returns: + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - Encoded output tensor + - Output lengths tensor + - Optional updated encoder states + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Example: + ```python + class ConcreteEncoder(AbsEncoder): + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + # Example implementation + encoded_output = self.encode(xs_pad) + output_lengths = self.calculate_lengths(ilens) + new_states = self.update_states(prev_states) + return encoded_output, output_lengths, new_states + + encoder = ConcreteEncoder() + output, lengths, states = encoder.forward(input_tensor, input_lengths) + ``` + """ raise NotImplementedError diff --git a/espnet2/asr/encoder/branchformer_encoder.py b/espnet2/asr/encoder/branchformer_encoder.py index 568a545d483..c1d474d0df0 100644 --- a/espnet2/asr/encoder/branchformer_encoder.py +++ b/espnet2/asr/encoder/branchformer_encoder.py @@ -47,19 +47,25 @@ class BranchformerEncoderLayer(torch.nn.Module): - """Branchformer encoder layer module. + """ + Branchformer encoder layer module. + + This layer combines self-attention and convolutional gating MLP branches + to capture both global and local context. Args: - size (int): model dimension - attn: standard self-attention or efficient attention, optional - cgmlp: ConvolutionalGatingMLP, optional - dropout_rate (float): dropout probability - merge_method (str): concat, learned_ave, fixed_ave - cgmlp_weight (float): weight of the cgmlp branch, between 0 and 1, - used if merge_method is fixed_ave - attn_branch_drop_rate (float): probability of dropping the attn branch, - used if merge_method is learned_ave - stochastic_depth_rate (float): stochastic depth probability + size (int): Model dimension. + attn (Optional[torch.nn.Module]): Self-attention module. + cgmlp (Optional[torch.nn.Module]): Convolutional Gating MLP module. + dropout_rate (float): Dropout probability. + merge_method (str): Method to merge branches ('concat', 'learned_ave', or 'fixed_ave'). + cgmlp_weight (float): Weight of the CGMLP branch for 'fixed_ave' merge method. Default: 0.5. + attn_branch_drop_rate (float): Probability of dropping the attention branch. Default: 0.0. + stochastic_depth_rate (float): Stochastic depth probability. Default: 0.0. + + Note: + At least one of `attn` or `cgmlp` must be provided. + The merge_method determines how the outputs of the two branches are combined. """ def __init__( @@ -136,18 +142,35 @@ def __init__( self.merge_proj = torch.nn.Identity() def forward(self, x_input, mask, cache=None): - """Compute encoded features. + """ + Compute encoded features. Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. + x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o positional embedding. - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - w/o pos emb: Tensor (#batch, time, size). mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). + cache (torch.Tensor, optional): Cache tensor of the input (#batch, time - 1, size). + Default: None. Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). + Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor]]: + - If positional embedding is used: + Tuple containing: + - torch.Tensor: Output tensor with positional embedding (#batch, time, size). + - torch.Tensor: Mask tensor (#batch, time). + - If positional embedding is not used: + Tuple containing: + - torch.Tensor: Output tensor (#batch, time, size). + - torch.Tensor: Mask tensor (#batch, time). + + Raises: + NotImplementedError: If cache is not None (caching is not implemented). + + Note: + This method applies the Branchformer encoding, combining self-attention + and convolutional gating MLP branches based on the specified merge method. + It also handles stochastic depth if enabled during training. """ if cache is not None: @@ -294,7 +317,42 @@ def forward(self, x_input, mask, cache=None): class BranchformerEncoder(AbsEncoder): - """Branchformer encoder module.""" + """ + Branchformer encoder module. + + This encoder combines self-attention and convolutional gating MLP in parallel branches + to capture both global and local context for speech recognition and understanding tasks. + + Args: + input_size (int): Dimension of input features. + output_size (int): Dimension of output features. Default: 256. + use_attn (bool): Whether to use attention branch. Default: True. + attention_heads (int): Number of attention heads. Default: 4. + attention_layer_type (str): Type of attention layer. Default: "rel_selfattn". + pos_enc_layer_type (str): Type of positional encoding. Default: "rel_pos". + rel_pos_type (str): Type of relative positional encoding. Default: "latest". + use_cgmlp (bool): Whether to use convolutional gating MLP branch. Default: True. + cgmlp_linear_units (int): Number of units in CGMLP linear layers. Default: 2048. + cgmlp_conv_kernel (int): Kernel size for CGMLP convolution. Default: 31. + use_linear_after_conv (bool): Whether to use linear layer after convolution in CGMLP. Default: False. + gate_activation (str): Activation function for CGMLP gate. Default: "identity". + merge_method (str): Method to merge branches. Default: "concat". + cgmlp_weight (Union[float, List[float]]): Weight(s) for CGMLP branch. Default: 0.5. + attn_branch_drop_rate (Union[float, List[float]]): Dropout rate(s) for attention branch. Default: 0.0. + num_blocks (int): Number of encoder blocks. Default: 12. + dropout_rate (float): Dropout rate. Default: 0.1. + positional_dropout_rate (float): Dropout rate for positional encoding. Default: 0.1. + attention_dropout_rate (float): Dropout rate for attention. Default: 0.0. + input_layer (Optional[str]): Type of input layer. Default: "conv2d". + zero_triu (bool): Whether to zero out the upper triangular part of attention matrix. Default: False. + padding_idx (int): Padding index for embedding layer. Default: -1. + stochastic_depth_rate (Union[float, List[float]]): Stochastic depth rate(s). Default: 0.0. + + Note: + This implementation is based on the paper "Branchformer: Parallel MLP-Attention + Architectures to Capture Local and Global Context for Speech Recognition and Understanding" + by Yifan Peng, et al. + """ @typechecked def __init__( @@ -505,6 +563,16 @@ def __init__( self.after_norm = LayerNorm(output_size) def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The dimension of the output features. + + Note: + This method returns the size of the encoder's output, + which is set during the initialization of the BranchformerEncoder. + """ return self._output_size def forward( @@ -513,18 +581,26 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. + """ + Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. + prev_states (torch.Tensor, optional): Not used in current implementation. Default: None. Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + - torch.Tensor: Output tensor (#batch, L, output_size). + - torch.Tensor: Output length (#batch). + - Optional[torch.Tensor]: Always None in current implementation. + + Raises: + TooShortUttError: If the input is too short for subsampling. + Note: + The method applies the Branchformer encoding to the input tensor, + handling potential subsampling and masking as needed. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/conformer_encoder.py b/espnet2/asr/encoder/conformer_encoder.py index 32f2134c561..856a57a8ec3 100644 --- a/espnet2/asr/encoder/conformer_encoder.py +++ b/espnet2/asr/encoder/conformer_encoder.py @@ -50,38 +50,31 @@ class ConformerEncoder(AbsEncoder): - """Conformer encoder module. - - Args: - input_size (int): Input dimension. - output_size (int): Dimension of attention. - attention_heads (int): The number of heads of multi head attention. - linear_units (int): The number of units of position-wise feed forward. - num_blocks (int): The number of decoder blocks. - dropout_rate (float): Dropout rate. - attention_dropout_rate (float): Dropout rate in attention. - positional_dropout_rate (float): Dropout rate after adding positional encoding. - input_layer (Union[str, torch.nn.Module]): Input layer type. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - If True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - If False, no additional linear will be applied. i.e. x -> x + att(x) - positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. - rel_pos_type (str): Whether to use the latest relative positional encoding or - the legacy one. The legacy relative positional encoding will be deprecated - in the future. More Details can be found in - https://github.com/espnet/espnet/pull/2816. - encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. - encoder_attn_layer_type (str): Encoder attention layer type. - activation_type (str): Encoder activation function type. - macaron_style (bool): Whether to use macaron style for positionwise layer. - use_cnn_module (bool): Whether to use convolution module. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - cnn_module_kernel (int): Kernerl size of convolution module. - padding_idx (int): Padding idx for input_layer=embed. - + """ + Conformer encoder module. + + This class implements the Conformer encoder, which combines self-attention and + convolution to model both global and local dependencies of an input sequence. + It is designed for speech recognition tasks and can be used as a powerful + feature extractor in various speech processing applications. + + The Conformer architecture integrates components from Transformers and + Convolutional Neural Networks (CNNs) to capture both long-range and local + dependencies in the input signal. It includes multi-headed self-attention + mechanisms, convolution modules, and feed-forward layers, along with + various normalization and regularization techniques. + + Key features of the ConformerEncoder include: + - Flexible input layer options (linear, conv2d, embed) + - Configurable number of encoder blocks + - Support for relative or absolute positional encodings + - Macaron-style feed-forward layers + - Optional convolution module in each block + - Stochastic depth and layer drop for regularization + - Intermediate CTC (Connectionist Temporal Classification) integration + + The encoder can be customized through numerous parameters to adapt to + different input sizes, model capacities, and specific task requirements. """ @typechecked @@ -301,6 +294,16 @@ def __init__( self.ctc_trim = ctc_trim def output_size(self) -> int: + """ + Returns the output size of the encoder. + + Returns: + int: The dimension of the encoder output. + + Note: + This method returns the value of the internal `_output_size` attribute, + which is typically set during the initialization of the encoder. + """ return self._output_size def forward( @@ -311,20 +314,34 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. + """ + Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - ctc (CTC): ctc module for intermediate CTC loss - return_all_hs (bool): whether to return all hidden states + prev_states (torch.Tensor, optional): Not used in current implementation. + ctc (CTC, optional): CTC module for intermediate CTC loss. + return_all_hs (bool, optional): Whether to return all hidden states. Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - torch.Tensor: Output tensor (#batch, L, output_size). + - torch.Tensor: Output length (#batch). + - Optional[torch.Tensor]: Not used in current implementation. + + Raises: + TooShortUttError: If the input is too short for subsampling. + + Note: + If intermediate CTC is used, the output tensor will be a tuple containing + the final output and a list of intermediate outputs. + + Examples: + >>> encoder = ConformerEncoder(input_size=80, output_size=256) + >>> x = torch.randn(2, 100, 80) + >>> ilens = torch.tensor([100, 80]) + >>> output, out_lens, _ = encoder(x, ilens) """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/contextual_block_conformer_encoder.py b/espnet2/asr/encoder/contextual_block_conformer_encoder.py index f8a761ac70a..bb4ea63ed78 100644 --- a/espnet2/asr/encoder/contextual_block_conformer_encoder.py +++ b/espnet2/asr/encoder/contextual_block_conformer_encoder.py @@ -1,596 +1,675 @@ -# -*- coding: utf-8 -*- -""" -Created on Sat Aug 21 17:27:16 2021. - -@author: Keqi Deng (UCAS) -""" - -import math -from typing import Optional, Tuple - -import torch -from typeguard import typechecked - -from espnet2.asr.encoder.abs_encoder import AbsEncoder -from espnet.nets.pytorch_backend.conformer.contextual_block_encoder_layer import ( - ContextualBlockEncoderLayer, -) -from espnet.nets.pytorch_backend.conformer.convolution import ConvolutionModule -from espnet.nets.pytorch_backend.nets_utils import get_activation, make_pad_mask -from espnet.nets.pytorch_backend.transformer.attention import MultiHeadedAttention -from espnet.nets.pytorch_backend.transformer.embedding import StreamPositionalEncoding -from espnet.nets.pytorch_backend.transformer.layer_norm import LayerNorm -from espnet.nets.pytorch_backend.transformer.multi_layer_conv import ( - Conv1dLinear, - MultiLayeredConv1d, -) -from espnet.nets.pytorch_backend.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from espnet.nets.pytorch_backend.transformer.repeat import repeat -from espnet.nets.pytorch_backend.transformer.subsampling_without_posenc import ( - Conv2dSubsamplingWOPosEnc, -) - - -class ContextualBlockConformerEncoder(AbsEncoder): - """Contextual Block Conformer encoder module. - - Args: - input_size: input dim - output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - positional_dropout_rate: dropout rate after adding positional encoding - input_layer: input layer type - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - positionwise_layer_type: linear of conv1d - positionwise_conv_kernel_size: kernel size of positionwise conv1d layer - padding_idx: padding_idx for input_layer=embed - block_size: block size for contextual block processing - hop_Size: hop size for block processing - look_ahead: look-ahead size for block_processing - init_average: whether to use average as initial context (otherwise max values) - ctx_pos_enc: whether to use positional encoding to the context vectors - """ - - @typechecked - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 3, - macaron_style: bool = False, - pos_enc_class=StreamPositionalEncoding, - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - cnn_module_kernel: int = 31, - padding_idx: int = -1, - block_size: int = 40, - hop_size: int = 16, - look_ahead: int = 16, - init_average: bool = True, - ctx_pos_enc: bool = True, - ): - super().__init__() - self._output_size = output_size - self.pos_enc = pos_enc_class(output_size, positional_dropout_rate) - activation = get_activation(activation_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - ) - self.subsample = 1 - elif input_layer == "conv2d": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, output_size, dropout_rate, kernels=[3, 3], strides=[2, 2] - ) - self.subsample = 4 - elif input_layer == "conv2d6": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, output_size, dropout_rate, kernels=[3, 5], strides=[2, 3] - ) - self.subsample = 6 - elif input_layer == "conv2d8": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, - output_size, - dropout_rate, - kernels=[3, 3, 3], - strides=[2, 2, 2], - ) - self.subsample = 8 - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - ) - self.subsample = 1 - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate), - ) - self.subsample = 1 - elif input_layer is None: - self.embed = torch.nn.Sequential( - pos_enc_class(output_size, positional_dropout_rate) - ) - self.subsample = 1 - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - convolution_layer = ConvolutionModule - convolution_layer_args = (output_size, cnn_module_kernel, activation) - - self.encoders = repeat( - num_blocks, - lambda lnum: ContextualBlockEncoderLayer( - output_size, - MultiHeadedAttention( - attention_heads, output_size, attention_dropout_rate - ), - positionwise_layer(*positionwise_layer_args), - positionwise_layer(*positionwise_layer_args) if macaron_style else None, - convolution_layer(*convolution_layer_args) if use_cnn_module else None, - dropout_rate, - num_blocks, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - # for block processing - self.block_size = block_size - self.hop_size = hop_size - self.look_ahead = look_ahead - self.init_average = init_average - self.ctx_pos_enc = ctx_pos_enc - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - is_final=True, - infer_mode=False, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - infer_mode: whether to be used for inference. This is used to - distinguish between forward_train (train and validate) and - forward_infer (decode). - Returns: - position embedded tensor and mask - """ - if self.training or not infer_mode: - return self.forward_train(xs_pad, ilens, prev_states) - else: - return self.forward_infer(xs_pad, ilens, prev_states, is_final) - - def forward_train( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): - xs_pad, masks = self.embed(xs_pad, masks) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - # create empty output container - total_frame_num = xs_pad.size(1) - ys_pad = xs_pad.new_zeros(xs_pad.size()) - - past_size = self.block_size - self.hop_size - self.look_ahead - - # block_size could be 0 meaning infinite - # apply usual encoder for short sequence - if self.block_size == 0 or total_frame_num <= self.block_size: - xs_pad, masks, _, _, _, _, _ = self.encoders( - self.pos_enc(xs_pad), masks, False, None, None - ) - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - return xs_pad, olens, None - - # start block processing - cur_hop = 0 - block_num = math.ceil( - float(total_frame_num - past_size - self.look_ahead) / float(self.hop_size) - ) - bsize = xs_pad.size(0) - addin = xs_pad.new_zeros( - bsize, block_num, xs_pad.size(-1) - ) # additional context embedding vecctors - - # first step - if self.init_average: # initialize with average value - addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).mean(1) - else: # initialize with max value - addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).max(1) - cur_hop += self.hop_size - # following steps - while cur_hop + self.block_size < total_frame_num: - if self.init_average: # initialize with average value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, self.block_size - ).mean(1) - else: # initialize with max value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, self.block_size - ).max(1) - cur_hop += self.hop_size - # last step - if cur_hop < total_frame_num and cur_hop // self.hop_size < block_num: - if self.init_average: # initialize with average value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, total_frame_num - cur_hop - ).mean(1) - else: # initialize with max value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, total_frame_num - cur_hop - ).max(1) - - if self.ctx_pos_enc: - addin = self.pos_enc(addin) - - xs_pad = self.pos_enc(xs_pad) - - # set up masks - mask_online = xs_pad.new_zeros( - xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 - ) - mask_online.narrow(2, 1, self.block_size + 1).narrow( - 3, 0, self.block_size + 1 - ).fill_(1) - - xs_chunk = xs_pad.new_zeros( - bsize, block_num, self.block_size + 2, xs_pad.size(-1) - ) - - # fill the input - # first step - left_idx = 0 - block_idx = 0 - xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( - -2, left_idx, self.block_size - ) - left_idx += self.hop_size - block_idx += 1 - # following steps - while left_idx + self.block_size < total_frame_num and block_idx < block_num: - xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( - -2, left_idx, self.block_size - ) - left_idx += self.hop_size - block_idx += 1 - # last steps - last_size = total_frame_num - left_idx - xs_chunk[:, block_idx, 1 : last_size + 1] = xs_pad.narrow( - -2, left_idx, last_size - ) - - # fill the initial context vector - xs_chunk[:, 0, 0] = addin[:, 0] - xs_chunk[:, 1:, 0] = addin[:, 0 : block_num - 1] - xs_chunk[:, :, self.block_size + 1] = addin - - # forward - ys_chunk, mask_online, _, _, _, _, _ = self.encoders( - xs_chunk, mask_online, False, xs_chunk - ) - - # copy output - # first step - offset = self.block_size - self.look_ahead - self.hop_size + 1 - left_idx = 0 - block_idx = 0 - cur_hop = self.block_size - self.look_ahead - ys_pad[:, left_idx:cur_hop] = ys_chunk[:, block_idx, 1 : cur_hop + 1] - left_idx += self.hop_size - block_idx += 1 - # following steps - while left_idx + self.block_size < total_frame_num and block_idx < block_num: - ys_pad[:, cur_hop : cur_hop + self.hop_size] = ys_chunk[ - :, block_idx, offset : offset + self.hop_size - ] - cur_hop += self.hop_size - left_idx += self.hop_size - block_idx += 1 - ys_pad[:, cur_hop:total_frame_num] = ys_chunk[ - :, block_idx, offset : last_size + 1, : - ] - - if self.normalize_before: - ys_pad = self.after_norm(ys_pad) - - olens = masks.squeeze(1).sum(1) - return ys_pad, olens, None - - def forward_infer( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - is_final: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - if prev_states is None: - prev_addin = None - buffer_before_downsampling = None - ilens_buffer = None - buffer_after_downsampling = None - n_processed_blocks = 0 - past_encoder_ctx = None - else: - prev_addin = prev_states["prev_addin"] - buffer_before_downsampling = prev_states["buffer_before_downsampling"] - ilens_buffer = prev_states["ilens_buffer"] - buffer_after_downsampling = prev_states["buffer_after_downsampling"] - n_processed_blocks = prev_states["n_processed_blocks"] - past_encoder_ctx = prev_states["past_encoder_ctx"] - bsize = xs_pad.size(0) - assert bsize == 1 - - if prev_states is not None: - xs_pad = torch.cat([buffer_before_downsampling, xs_pad], dim=1) - ilens += ilens_buffer - - if is_final: - buffer_before_downsampling = None - else: - n_samples = xs_pad.size(1) // self.subsample - 1 - if n_samples < 2: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": xs_pad, - "ilens_buffer": ilens, - "buffer_after_downsampling": buffer_after_downsampling, - "n_processed_blocks": n_processed_blocks, - "past_encoder_ctx": past_encoder_ctx, - } - return ( - xs_pad.new_zeros(bsize, 0, self._output_size), - xs_pad.new_zeros(bsize), - next_states, - ) - - n_res_samples = xs_pad.size(1) % self.subsample + self.subsample * 2 - buffer_before_downsampling = xs_pad.narrow( - 1, xs_pad.size(1) - n_res_samples, n_res_samples - ) - xs_pad = xs_pad.narrow(1, 0, n_samples * self.subsample) - - ilens_buffer = ilens.new_full( - [1], dtype=torch.long, fill_value=n_res_samples - ) - ilens = ilens.new_full( - [1], dtype=torch.long, fill_value=n_samples * self.subsample - ) - - if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): - xs_pad, _ = self.embed(xs_pad, None) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - # create empty output container - if buffer_after_downsampling is not None: - xs_pad = torch.cat([buffer_after_downsampling, xs_pad], dim=1) - - total_frame_num = xs_pad.size(1) - - if is_final: - past_size = self.block_size - self.hop_size - self.look_ahead - block_num = math.ceil( - float(total_frame_num - past_size - self.look_ahead) - / float(self.hop_size) - ) - buffer_after_downsampling = None - else: - if total_frame_num <= self.block_size: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": buffer_before_downsampling, - "ilens_buffer": ilens_buffer, - "buffer_after_downsampling": xs_pad, - "n_processed_blocks": n_processed_blocks, - "past_encoder_ctx": past_encoder_ctx, - } - return ( - xs_pad.new_zeros(bsize, 0, self._output_size), - xs_pad.new_zeros(bsize), - next_states, - ) - - overlap_size = self.block_size - self.hop_size - block_num = max(0, xs_pad.size(1) - overlap_size) // self.hop_size - res_frame_num = xs_pad.size(1) - self.hop_size * block_num - buffer_after_downsampling = xs_pad.narrow( - 1, xs_pad.size(1) - res_frame_num, res_frame_num - ) - xs_pad = xs_pad.narrow(1, 0, block_num * self.hop_size + overlap_size) - - # block_size could be 0 meaning infinite - # apply usual encoder for short sequence - assert self.block_size > 0 - if n_processed_blocks == 0 and total_frame_num <= self.block_size and is_final: - xs_chunk = self.pos_enc(xs_pad).unsqueeze(1) - xs_pad, _, _, _, _, _, _ = self.encoders( - xs_chunk, None, True, None, None, True - ) - xs_pad = xs_pad.squeeze(0) - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - return xs_pad, xs_pad.new_zeros(bsize), None - # return xs_pad, None, None - - # start block processing - xs_chunk = xs_pad.new_zeros( - bsize, block_num, self.block_size + 2, xs_pad.size(-1) - ) - - for i in range(block_num): - cur_hop = i * self.hop_size - chunk_length = min(self.block_size, total_frame_num - cur_hop) - addin = xs_pad.narrow(1, cur_hop, chunk_length) - if self.init_average: - addin = addin.mean(1, keepdim=True) - else: - addin = addin.max(1, keepdim=True) - if self.ctx_pos_enc: - addin = self.pos_enc(addin, i + n_processed_blocks) - - if prev_addin is None: - prev_addin = addin - xs_chunk[:, i, 0] = prev_addin - xs_chunk[:, i, -1] = addin - - chunk = self.pos_enc( - xs_pad.narrow(1, cur_hop, chunk_length), - cur_hop + self.hop_size * n_processed_blocks, - ) - - xs_chunk[:, i, 1 : chunk_length + 1] = chunk - - prev_addin = addin - - # mask setup, it should be the same to that of forward_train - mask_online = xs_pad.new_zeros( - xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 - ) - mask_online.narrow(2, 1, self.block_size + 1).narrow( - 3, 0, self.block_size + 1 - ).fill_(1) - - ys_chunk, _, _, _, past_encoder_ctx, _, _ = self.encoders( - xs_chunk, mask_online, True, past_encoder_ctx - ) - - # remove addin - ys_chunk = ys_chunk.narrow(2, 1, self.block_size) - - offset = self.block_size - self.look_ahead - self.hop_size - if is_final: - if n_processed_blocks == 0: - y_length = xs_pad.size(1) - else: - y_length = xs_pad.size(1) - offset - else: - y_length = block_num * self.hop_size - if n_processed_blocks == 0: - y_length += offset - ys_pad = xs_pad.new_zeros((xs_pad.size(0), y_length, xs_pad.size(2))) - if n_processed_blocks == 0: - ys_pad[:, 0:offset] = ys_chunk[:, 0, 0:offset] - for i in range(block_num): - cur_hop = i * self.hop_size - if n_processed_blocks == 0: - cur_hop += offset - if i == block_num - 1 and is_final: - chunk_length = min(self.block_size - offset, ys_pad.size(1) - cur_hop) - else: - chunk_length = self.hop_size - ys_pad[:, cur_hop : cur_hop + chunk_length] = ys_chunk[ - :, i, offset : offset + chunk_length - ] - if self.normalize_before: - ys_pad = self.after_norm(ys_pad) - - if is_final: - next_states = None - else: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": buffer_before_downsampling, - "ilens_buffer": ilens_buffer, - "buffer_after_downsampling": buffer_after_downsampling, - "n_processed_blocks": n_processed_blocks + block_num, - "past_encoder_ctx": past_encoder_ctx, - } - - return ( - ys_pad, - torch.tensor([y_length], dtype=xs_pad.dtype, device=ys_pad.device), - next_states, - ) - # return ys_pad, None, next_states +# -*- coding: utf-8 -*- +""" +Created on Sat Aug 21 17:27:16 2021. + +@author: Keqi Deng (UCAS) +""" + +import math +from typing import Optional, Tuple + +import torch +from typeguard import typechecked + +from espnet2.asr.encoder.abs_encoder import AbsEncoder +from espnet.nets.pytorch_backend.conformer.contextual_block_encoder_layer import ( + ContextualBlockEncoderLayer, +) +from espnet.nets.pytorch_backend.conformer.convolution import ConvolutionModule +from espnet.nets.pytorch_backend.nets_utils import get_activation, make_pad_mask +from espnet.nets.pytorch_backend.transformer.attention import MultiHeadedAttention +from espnet.nets.pytorch_backend.transformer.embedding import StreamPositionalEncoding +from espnet.nets.pytorch_backend.transformer.layer_norm import LayerNorm +from espnet.nets.pytorch_backend.transformer.multi_layer_conv import ( + Conv1dLinear, + MultiLayeredConv1d, +) +from espnet.nets.pytorch_backend.transformer.positionwise_feed_forward import ( + PositionwiseFeedForward, +) +from espnet.nets.pytorch_backend.transformer.repeat import repeat +from espnet.nets.pytorch_backend.transformer.subsampling_without_posenc import ( + Conv2dSubsamplingWOPosEnc, +) + + +class ContextualBlockConformerEncoder(AbsEncoder): + """ + Contextual Block Conformer encoder module. + + This class implements a Conformer encoder with contextual block processing, + which allows for efficient processing of long sequences by dividing them + into smaller blocks and incorporating context information. + + Args: + input_size (int): Dimension of input features. + output_size (int, optional): Dimension of attention and output features. Defaults to 256. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in position-wise feed forward layers. Defaults to 2048. + num_blocks (int, optional): Number of encoder blocks. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. + input_layer (str, optional): Type of input layer. Defaults to "conv2d". + normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to True. + concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. + positionwise_layer_type (str, optional): Type of position-wise layer. Defaults to "linear". + positionwise_conv_kernel_size (int, optional): Kernel size of position-wise conv1d layer. Defaults to 3. + macaron_style (bool, optional): Whether to use macaron style for positionwise layer. Defaults to False. + pos_enc_class (class, optional): Positional encoding class. Defaults to StreamPositionalEncoding. + selfattention_layer_type (str, optional): Type of self-attention layer. Defaults to "rel_selfattn". + activation_type (str, optional): Type of activation function. Defaults to "swish". + use_cnn_module (bool, optional): Whether to use CNN module. Defaults to True. + cnn_module_kernel (int, optional): Kernel size of CNN module. Defaults to 31. + padding_idx (int, optional): Padding index for input layer. Defaults to -1. + block_size (int, optional): Block size for contextual block processing. Defaults to 40. + hop_size (int, optional): Hop size for block processing. Defaults to 16. + look_ahead (int, optional): Look-ahead size for block processing. Defaults to 16. + init_average (bool, optional): Whether to use average as initial context. Defaults to True. + ctx_pos_enc (bool, optional): Whether to use positional encoding for context vectors. Defaults to True. + + Attributes: + block_size (int): Block size for contextual block processing. + hop_size (int): Hop size for block processing. + look_ahead (int): Look-ahead size for block processing. + init_average (bool): Whether to use average as initial context. + ctx_pos_enc (bool): Whether to use positional encoding for context vectors. + """ + + @typechecked + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 3, + macaron_style: bool = False, + pos_enc_class=StreamPositionalEncoding, + selfattention_layer_type: str = "rel_selfattn", + activation_type: str = "swish", + use_cnn_module: bool = True, + cnn_module_kernel: int = 31, + padding_idx: int = -1, + block_size: int = 40, + hop_size: int = 16, + look_ahead: int = 16, + init_average: bool = True, + ctx_pos_enc: bool = True, + ): + super().__init__() + self._output_size = output_size + self.pos_enc = pos_enc_class(output_size, positional_dropout_rate) + activation = get_activation(activation_type) + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + ) + self.subsample = 1 + elif input_layer == "conv2d": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, output_size, dropout_rate, kernels=[3, 3], strides=[2, 2] + ) + self.subsample = 4 + elif input_layer == "conv2d6": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, output_size, dropout_rate, kernels=[3, 5], strides=[2, 3] + ) + self.subsample = 6 + elif input_layer == "conv2d8": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, + output_size, + dropout_rate, + kernels=[3, 3, 3], + strides=[2, 2, 2], + ) + self.subsample = 8 + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + ) + self.subsample = 1 + elif isinstance(input_layer, torch.nn.Module): + self.embed = torch.nn.Sequential( + input_layer, + pos_enc_class(output_size, positional_dropout_rate), + ) + self.subsample = 1 + elif input_layer is None: + self.embed = torch.nn.Sequential( + pos_enc_class(output_size, positional_dropout_rate) + ) + self.subsample = 1 + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + convolution_layer = ConvolutionModule + convolution_layer_args = (output_size, cnn_module_kernel, activation) + + self.encoders = repeat( + num_blocks, + lambda lnum: ContextualBlockEncoderLayer( + output_size, + MultiHeadedAttention( + attention_heads, output_size, attention_dropout_rate + ), + positionwise_layer(*positionwise_layer_args), + positionwise_layer(*positionwise_layer_args) if macaron_style else None, + convolution_layer(*convolution_layer_args) if use_cnn_module else None, + dropout_rate, + num_blocks, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + # for block processing + self.block_size = block_size + self.hop_size = hop_size + self.look_ahead = look_ahead + self.init_average = init_average + self.ctx_pos_enc = ctx_pos_enc + + def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The output size (dimension) of the encoder. + """ + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + is_final=True, + infer_mode=False, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Forward pass of the Contextual Block Conformer Encoder. + + This method processes the input tensor and returns the encoded output. It handles both + training/validation and inference modes. + + Args: + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, + L is sequence length, and D is input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + prev_states (torch.Tensor, optional): Previous encoder states for streaming inference. + Defaults to None. + is_final (bool, optional): Whether this is the final block in streaming inference. + Defaults to True. + infer_mode (bool, optional): Whether to use inference mode. Defaults to False. + + Returns: + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - Encoded output tensor of shape (B, L, D_out), where D_out is the output dimension. + - Output lengths of shape (B,). + - Updated encoder states (None if not in inference mode). + + Note: + The behavior of this method differs between training/validation and inference modes. + In training/validation, it processes the entire input sequence at once. + In inference mode, it supports streaming processing of input chunks. + """ + if self.training or not infer_mode: + return self.forward_train(xs_pad, ilens, prev_states) + else: + return self.forward_infer(xs_pad, ilens, prev_states, is_final) + + def forward_train( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Forward pass for training and validation. + + This method processes the entire input sequence at once, applying the contextual + block processing technique for long sequences. + + Args: + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, + L is sequence length, and D is input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + prev_states (torch.Tensor, optional): Not used in training mode. Defaults to None. + + Returns: + Tuple[torch.Tensor, torch.Tensor, None]: A tuple containing: + - Encoded output tensor of shape (B, L, D_out), where D_out is the output dimension. + - Output lengths of shape (B,). + - None (placeholder for consistency with inference mode). + + Note: + This method implements the core contextual block processing algorithm, including: + - Subsampling and positional encoding of input + - Creation and processing of context vectors + - Block-wise processing of the input sequence + - Aggregation of block outputs into the final encoded sequence + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + + if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): + xs_pad, masks = self.embed(xs_pad, masks) + elif self.embed is not None: + xs_pad = self.embed(xs_pad) + + # create empty output container + total_frame_num = xs_pad.size(1) + ys_pad = xs_pad.new_zeros(xs_pad.size()) + + past_size = self.block_size - self.hop_size - self.look_ahead + + # block_size could be 0 meaning infinite + # apply usual encoder for short sequence + if self.block_size == 0 or total_frame_num <= self.block_size: + xs_pad, masks, _, _, _, _, _ = self.encoders( + self.pos_enc(xs_pad), masks, False, None, None + ) + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + return xs_pad, olens, None + + # start block processing + cur_hop = 0 + block_num = math.ceil( + float(total_frame_num - past_size - self.look_ahead) / float(self.hop_size) + ) + bsize = xs_pad.size(0) + addin = xs_pad.new_zeros( + bsize, block_num, xs_pad.size(-1) + ) # additional context embedding vecctors + + # first step + if self.init_average: # initialize with average value + addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).mean(1) + else: # initialize with max value + addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).max(1) + cur_hop += self.hop_size + # following steps + while cur_hop + self.block_size < total_frame_num: + if self.init_average: # initialize with average value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, self.block_size + ).mean(1) + else: # initialize with max value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, self.block_size + ).max(1) + cur_hop += self.hop_size + # last step + if cur_hop < total_frame_num and cur_hop // self.hop_size < block_num: + if self.init_average: # initialize with average value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, total_frame_num - cur_hop + ).mean(1) + else: # initialize with max value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, total_frame_num - cur_hop + ).max(1) + + if self.ctx_pos_enc: + addin = self.pos_enc(addin) + + xs_pad = self.pos_enc(xs_pad) + + # set up masks + mask_online = xs_pad.new_zeros( + xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 + ) + mask_online.narrow(2, 1, self.block_size + 1).narrow( + 3, 0, self.block_size + 1 + ).fill_(1) + + xs_chunk = xs_pad.new_zeros( + bsize, block_num, self.block_size + 2, xs_pad.size(-1) + ) + + # fill the input + # first step + left_idx = 0 + block_idx = 0 + xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( + -2, left_idx, self.block_size + ) + left_idx += self.hop_size + block_idx += 1 + # following steps + while left_idx + self.block_size < total_frame_num and block_idx < block_num: + xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( + -2, left_idx, self.block_size + ) + left_idx += self.hop_size + block_idx += 1 + # last steps + last_size = total_frame_num - left_idx + xs_chunk[:, block_idx, 1 : last_size + 1] = xs_pad.narrow( + -2, left_idx, last_size + ) + + # fill the initial context vector + xs_chunk[:, 0, 0] = addin[:, 0] + xs_chunk[:, 1:, 0] = addin[:, 0 : block_num - 1] + xs_chunk[:, :, self.block_size + 1] = addin + + # forward + ys_chunk, mask_online, _, _, _, _, _ = self.encoders( + xs_chunk, mask_online, False, xs_chunk + ) + + # copy output + # first step + offset = self.block_size - self.look_ahead - self.hop_size + 1 + left_idx = 0 + block_idx = 0 + cur_hop = self.block_size - self.look_ahead + ys_pad[:, left_idx:cur_hop] = ys_chunk[:, block_idx, 1 : cur_hop + 1] + left_idx += self.hop_size + block_idx += 1 + # following steps + while left_idx + self.block_size < total_frame_num and block_idx < block_num: + ys_pad[:, cur_hop : cur_hop + self.hop_size] = ys_chunk[ + :, block_idx, offset : offset + self.hop_size + ] + cur_hop += self.hop_size + left_idx += self.hop_size + block_idx += 1 + ys_pad[:, cur_hop:total_frame_num] = ys_chunk[ + :, block_idx, offset : last_size + 1, : + ] + + if self.normalize_before: + ys_pad = self.after_norm(ys_pad) + + olens = masks.squeeze(1).sum(1) + return ys_pad, olens, None + + def forward_infer( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + is_final: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Forward pass for inference (decoding) with streaming support. + + This method processes input chunks sequentially, maintaining necessary states + for continuous streaming inference. + + Args: + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, + L is sequence length, and D is input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + prev_states (torch.Tensor, optional): Previous encoder states from the last call. + Contains buffers and processing information. Defaults to None. + is_final (bool, optional): Whether this is the final chunk of the utterance. + Defaults to True. + + Returns: + Tuple[torch.Tensor, torch.Tensor, Optional[Dict]]: A tuple containing: + - Encoded output tensor of shape (B, L', D_out), where L' is the processed + length and D_out is the output dimension. + - Output lengths of shape (B,). + - Updated encoder states (Dict) for the next call, or None if is_final is True. + + Note: + This method implements streaming inference by: + - Managing buffers for input and output + - Applying block-wise processing on the input chunk + - Handling overlap between consecutive chunks + - Maintaining context information across calls + - Supporting final processing when the last chunk is received + + The returned states dict includes: + - prev_addin: Previous additional input for context + - buffer_before_downsampling: Input buffer before downsampling + - ilens_buffer: Input length buffer + - buffer_after_downsampling: Input buffer after downsampling + - n_processed_blocks: Number of processed blocks + - past_encoder_ctx: Past encoder context + """ + if prev_states is None: + prev_addin = None + buffer_before_downsampling = None + ilens_buffer = None + buffer_after_downsampling = None + n_processed_blocks = 0 + past_encoder_ctx = None + else: + prev_addin = prev_states["prev_addin"] + buffer_before_downsampling = prev_states["buffer_before_downsampling"] + ilens_buffer = prev_states["ilens_buffer"] + buffer_after_downsampling = prev_states["buffer_after_downsampling"] + n_processed_blocks = prev_states["n_processed_blocks"] + past_encoder_ctx = prev_states["past_encoder_ctx"] + bsize = xs_pad.size(0) + assert bsize == 1 + + if prev_states is not None: + xs_pad = torch.cat([buffer_before_downsampling, xs_pad], dim=1) + ilens += ilens_buffer + + if is_final: + buffer_before_downsampling = None + else: + n_samples = xs_pad.size(1) // self.subsample - 1 + if n_samples < 2: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": xs_pad, + "ilens_buffer": ilens, + "buffer_after_downsampling": buffer_after_downsampling, + "n_processed_blocks": n_processed_blocks, + "past_encoder_ctx": past_encoder_ctx, + } + return ( + xs_pad.new_zeros(bsize, 0, self._output_size), + xs_pad.new_zeros(bsize), + next_states, + ) + + n_res_samples = xs_pad.size(1) % self.subsample + self.subsample * 2 + buffer_before_downsampling = xs_pad.narrow( + 1, xs_pad.size(1) - n_res_samples, n_res_samples + ) + xs_pad = xs_pad.narrow(1, 0, n_samples * self.subsample) + + ilens_buffer = ilens.new_full( + [1], dtype=torch.long, fill_value=n_res_samples + ) + ilens = ilens.new_full( + [1], dtype=torch.long, fill_value=n_samples * self.subsample + ) + + if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): + xs_pad, _ = self.embed(xs_pad, None) + elif self.embed is not None: + xs_pad = self.embed(xs_pad) + + # create empty output container + if buffer_after_downsampling is not None: + xs_pad = torch.cat([buffer_after_downsampling, xs_pad], dim=1) + + total_frame_num = xs_pad.size(1) + + if is_final: + past_size = self.block_size - self.hop_size - self.look_ahead + block_num = math.ceil( + float(total_frame_num - past_size - self.look_ahead) + / float(self.hop_size) + ) + buffer_after_downsampling = None + else: + if total_frame_num <= self.block_size: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": buffer_before_downsampling, + "ilens_buffer": ilens_buffer, + "buffer_after_downsampling": xs_pad, + "n_processed_blocks": n_processed_blocks, + "past_encoder_ctx": past_encoder_ctx, + } + return ( + xs_pad.new_zeros(bsize, 0, self._output_size), + xs_pad.new_zeros(bsize), + next_states, + ) + + overlap_size = self.block_size - self.hop_size + block_num = max(0, xs_pad.size(1) - overlap_size) // self.hop_size + res_frame_num = xs_pad.size(1) - self.hop_size * block_num + buffer_after_downsampling = xs_pad.narrow( + 1, xs_pad.size(1) - res_frame_num, res_frame_num + ) + xs_pad = xs_pad.narrow(1, 0, block_num * self.hop_size + overlap_size) + + # block_size could be 0 meaning infinite + # apply usual encoder for short sequence + assert self.block_size > 0 + if n_processed_blocks == 0 and total_frame_num <= self.block_size and is_final: + xs_chunk = self.pos_enc(xs_pad).unsqueeze(1) + xs_pad, _, _, _, _, _, _ = self.encoders( + xs_chunk, None, True, None, None, True + ) + xs_pad = xs_pad.squeeze(0) + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + return xs_pad, xs_pad.new_zeros(bsize), None + # return xs_pad, None, None + + # start block processing + xs_chunk = xs_pad.new_zeros( + bsize, block_num, self.block_size + 2, xs_pad.size(-1) + ) + + for i in range(block_num): + cur_hop = i * self.hop_size + chunk_length = min(self.block_size, total_frame_num - cur_hop) + addin = xs_pad.narrow(1, cur_hop, chunk_length) + if self.init_average: + addin = addin.mean(1, keepdim=True) + else: + addin = addin.max(1, keepdim=True) + if self.ctx_pos_enc: + addin = self.pos_enc(addin, i + n_processed_blocks) + + if prev_addin is None: + prev_addin = addin + xs_chunk[:, i, 0] = prev_addin + xs_chunk[:, i, -1] = addin + + chunk = self.pos_enc( + xs_pad.narrow(1, cur_hop, chunk_length), + cur_hop + self.hop_size * n_processed_blocks, + ) + + xs_chunk[:, i, 1 : chunk_length + 1] = chunk + + prev_addin = addin + + # mask setup, it should be the same to that of forward_train + mask_online = xs_pad.new_zeros( + xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 + ) + mask_online.narrow(2, 1, self.block_size + 1).narrow( + 3, 0, self.block_size + 1 + ).fill_(1) + + ys_chunk, _, _, _, past_encoder_ctx, _, _ = self.encoders( + xs_chunk, mask_online, True, past_encoder_ctx + ) + + # remove addin + ys_chunk = ys_chunk.narrow(2, 1, self.block_size) + + offset = self.block_size - self.look_ahead - self.hop_size + if is_final: + if n_processed_blocks == 0: + y_length = xs_pad.size(1) + else: + y_length = xs_pad.size(1) - offset + else: + y_length = block_num * self.hop_size + if n_processed_blocks == 0: + y_length += offset + ys_pad = xs_pad.new_zeros((xs_pad.size(0), y_length, xs_pad.size(2))) + if n_processed_blocks == 0: + ys_pad[:, 0:offset] = ys_chunk[:, 0, 0:offset] + for i in range(block_num): + cur_hop = i * self.hop_size + if n_processed_blocks == 0: + cur_hop += offset + if i == block_num - 1 and is_final: + chunk_length = min(self.block_size - offset, ys_pad.size(1) - cur_hop) + else: + chunk_length = self.hop_size + ys_pad[:, cur_hop : cur_hop + chunk_length] = ys_chunk[ + :, i, offset : offset + chunk_length + ] + if self.normalize_before: + ys_pad = self.after_norm(ys_pad) + + if is_final: + next_states = None + else: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": buffer_before_downsampling, + "ilens_buffer": ilens_buffer, + "buffer_after_downsampling": buffer_after_downsampling, + "n_processed_blocks": n_processed_blocks + block_num, + "past_encoder_ctx": past_encoder_ctx, + } + + return ( + ys_pad, + torch.tensor([y_length], dtype=xs_pad.dtype, device=ys_pad.device), + next_states, + ) + # return ys_pad, None, next_states diff --git a/espnet2/asr/encoder/contextual_block_transformer_encoder.py b/espnet2/asr/encoder/contextual_block_transformer_encoder.py index a0732c8cd1f..3437a18ffcf 100644 --- a/espnet2/asr/encoder/contextual_block_transformer_encoder.py +++ b/espnet2/asr/encoder/contextual_block_transformer_encoder.py @@ -30,36 +30,17 @@ class ContextualBlockTransformerEncoder(AbsEncoder): - """Contextual Block Transformer encoder module. - - Details in Tsunoo et al. "Transformer ASR with contextual block processing" - (https://arxiv.org/abs/1910.07204) - - Args: - input_size: input dim - output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of encoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - positional_dropout_rate: dropout rate after adding positional encoding - input_layer: input layer type - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - positionwise_layer_type: linear of conv1d - positionwise_conv_kernel_size: kernel size of positionwise conv1d layer - padding_idx: padding_idx for input_layer=embed - block_size: block size for contextual block processing - hop_Size: hop size for block processing - look_ahead: look-ahead size for block_processing - init_average: whether to use average as initial context (otherwise max values) - ctx_pos_enc: whether to use positional encoding to the context vectors + """ + Contextual Block Transformer encoder module. + + This class implements the Contextual Block Transformer encoder as described in + Tsunoo et al. "Transformer ASR with contextual block processing" + (https://arxiv.org/abs/1910.07204). It processes input in blocks for efficient + streaming ASR and long-form audio processing. + + The encoder uses a combination of self-attention mechanisms and feed-forward + networks, with the addition of contextual block processing to handle long + sequences efficiently. """ @typechecked @@ -179,6 +160,12 @@ def __init__( self.ctx_pos_enc = ctx_pos_enc def output_size(self) -> int: + """ + Return the output size of the encoder. + + Returns: + int: The dimension of the output features. + """ return self._output_size def forward( @@ -189,17 +176,24 @@ def forward( is_final=True, infer_mode=False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Process the input tensor and return the encoded output. + + This method handles both training and inference modes. For inference, + it uses block processing to handle streaming input efficiently. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - infer_mode: whether to be used for inference. This is used to - distinguish between forward_train (train and validate) and - forward_infer (decode). + xs_pad (torch.Tensor): Input tensor (B, L, D) + ilens (torch.Tensor): Input length (B) + prev_states (torch.Tensor, optional): Previous states for streaming inference. Defaults to None. + is_final (bool, optional): Whether this is the final block in streaming inference. Defaults to True. + infer_mode (bool, optional): Whether to use inference mode. Defaults to False. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + - Encoded output tensor + - Output lengths + - Optional next states for streaming inference """ if self.training or not infer_mode: return self.forward_train(xs_pad, ilens, prev_states) @@ -212,14 +206,22 @@ def forward_train( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Process the input tensor for training or validation. + + This method applies the full contextual block transformer encoding process + to the input, suitable for training and validation phases. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor (B, L, D) + ilens (torch.Tensor): Input length (B) + prev_states (torch.Tensor, optional): Not used in training mode. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + - Encoded output tensor + - Output lengths + - None (no states are returned in training mode) """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) @@ -367,14 +369,24 @@ def forward_infer( prev_states: torch.Tensor = None, is_final: bool = True, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Process the input tensor for inference in streaming mode. + + This method applies the contextual block transformer encoding process + to the input, optimized for streaming inference. It handles partial inputs + and maintains state between calls for continuous processing. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor (B, L, D) + ilens (torch.Tensor): Input length (B) + prev_states (torch.Tensor, optional): Previous states from the last call. Defaults to None. + is_final (bool, optional): Whether this is the final block in the stream. Defaults to True. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + - Encoded output tensor + - None (output lengths are not computed in inference mode) + - Next states to be used in the subsequent call, or None if is_final is True """ if prev_states is None: prev_addin = None diff --git a/espnet2/asr/encoder/e_branchformer_encoder.py b/espnet2/asr/encoder/e_branchformer_encoder.py index ae2381c234e..abdd3cfbe2a 100644 --- a/espnet2/asr/encoder/e_branchformer_encoder.py +++ b/espnet2/asr/encoder/e_branchformer_encoder.py @@ -51,16 +51,40 @@ class EBranchformerEncoderLayer(torch.nn.Module): - """E-Branchformer encoder layer module. + """ + E-Branchformer encoder layer module. + + This class implements a single layer of the E-Branchformer encoder, which combines + self-attention and convolutional gating MLP branches. Args: - size (int): model dimension - attn: standard self-attention or efficient attention - cgmlp: ConvolutionalGatingMLP - feed_forward: feed-forward module, optional - feed_forward: macaron-style feed-forward module, optional - dropout_rate (float): dropout probability - merge_conv_kernel (int): kernel size of the depth-wise conv in merge module + size (int): Model dimension. + attn (torch.nn.Module): Self-attention module. + cgmlp (torch.nn.Module): Convolutional Gating MLP module. + feed_forward (Optional[torch.nn.Module]): Feed-forward module. + feed_forward_macaron (Optional[torch.nn.Module]): Macaron-style feed-forward module. + dropout_rate (float): Dropout probability. + merge_conv_kernel (int): Kernel size of the depth-wise conv in merge module. Defaults to 3. + + Attributes: + size (int): Model dimension. + attn (torch.nn.Module): Self-attention module. + cgmlp (torch.nn.Module): Convolutional Gating MLP module. + feed_forward (Optional[torch.nn.Module]): Feed-forward module. + feed_forward_macaron (Optional[torch.nn.Module]): Macaron-style feed-forward module. + ff_scale (float): Scaling factor for feed-forward modules. + norm_ff (LayerNorm): Layer normalization for feed-forward module. + norm_ff_macaron (LayerNorm): Layer normalization for macaron-style feed-forward module. + norm_mha (LayerNorm): Layer normalization for multi-head attention module. + norm_mlp (LayerNorm): Layer normalization for MLP module. + norm_final (LayerNorm): Final layer normalization. + dropout (torch.nn.Dropout): Dropout layer. + depthwise_conv_fusion (torch.nn.Conv1d): Depth-wise convolution for branch fusion. + merge_proj (torch.nn.Linear): Linear projection for branch merging. + + Note: + This implementation is based on the paper "E-Branchformer: Branchformer with + Enhanced merging for speech recognition" by Kim et al. """ def __init__( @@ -106,17 +130,31 @@ def __init__( self.merge_proj = torch.nn.Linear(size + size, size) def forward(self, x_input, mask, cache=None): - """Compute encoded features. + """ + Compute encoded features. + + This method processes the input through the E-Branchformer encoder layer, + including self-attention and convolutional gating MLP branches. Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. + x_input (Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]): Input tensor w/ or w/o positional embedding. - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - w/o pos emb: Tensor (#batch, time, size). mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). + cache (Optional[torch.Tensor]): Cache tensor of the input (#batch, time - 1, size). Defaults to None. + Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). + Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: Output tensor or tuple of tensors. + - If input includes positional embedding: ((#batch, time, size), (1, time, size)). + - If input doesn't include positional embedding: (#batch, time, size). + torch.Tensor: Updated mask tensor (#batch, time). + + Raises: + NotImplementedError: If cache is not None, which is currently not supported. + + Note: + The method applies various transformations including feed-forward layers, + self-attention, convolutional gating MLP, and branch merging. """ if cache is not None: @@ -182,7 +220,53 @@ def forward(self, x_input, mask, cache=None): class EBranchformerEncoder(AbsEncoder): - """E-Branchformer encoder module.""" + """ + E-Branchformer encoder module. + + This class implements the E-Branchformer encoder, which combines self-attention + and convolutional gating MLP in a branched structure for speech recognition tasks. + + Args: + input_size (int): Dimension of input features. + output_size (int): Dimension of output features. Defaults to 256. + attention_heads (int): Number of attention heads. Defaults to 4. + attention_layer_type (str): Type of attention layer. Defaults to "rel_selfattn". + pos_enc_layer_type (str): Type of positional encoding layer. Defaults to "rel_pos". + rel_pos_type (str): Type of relative positional encoding. Defaults to "latest". + cgmlp_linear_units (int): Number of units in CGMLP linear layer. Defaults to 2048. + cgmlp_conv_kernel (int): Kernel size for CGMLP convolution. Defaults to 31. + use_linear_after_conv (bool): Whether to use linear layer after convolution. Defaults to False. + gate_activation (str): Activation function for the gate. Defaults to "identity". + num_blocks (int): Number of encoder blocks. Defaults to 12. + dropout_rate (float): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float): Dropout rate for positional encoding. Defaults to 0.1. + attention_dropout_rate (float): Dropout rate for attention. Defaults to 0.0. + input_layer (Optional[str]): Type of input layer. Defaults to "conv2d". + zero_triu (bool): Whether to zero out the upper triangular part of attention matrix. Defaults to False. + padding_idx (int): Padding index for embedding layer. Defaults to -1. + layer_drop_rate (float): Layer dropout rate. Defaults to 0.0. + max_pos_emb_len (int): Maximum length for positional embedding. Defaults to 5000. + use_ffn (bool): Whether to use feed-forward network. Defaults to False. + macaron_ffn (bool): Whether to use macaron-style feed-forward network. Defaults to False. + ffn_activation_type (str): Activation function for feed-forward network. Defaults to "swish". + linear_units (int): Number of units in feed-forward network. Defaults to 2048. + positionwise_layer_type (str): Type of position-wise layer. Defaults to "linear". + merge_conv_kernel (int): Kernel size for merge convolution. Defaults to 3. + interctc_layer_idx (Optional[List[int]]): Indices of layers to apply intermediate CTC. Defaults to None. + interctc_use_conditioning (bool): Whether to use intermediate CTC for conditioning. Defaults to False. + + Attributes: + embed (torch.nn.Module): Input embedding layer. + encoders (torch.nn.Module): Sequence of encoder layers. + after_norm (LayerNorm): Layer normalization after the encoder layers. + interctc_layer_idx (List[int]): Indices of layers to apply intermediate CTC. + interctc_use_conditioning (bool): Whether to use intermediate CTC for conditioning. + conditioning_layer (Optional[torch.nn.Module]): Conditioning layer for intermediate CTC. + + Note: + This implementation is based on the paper "E-Branchformer: Branchformer with + Enhanced merging for speech recognition" by Kim et al. + """ @typechecked def __init__( @@ -418,6 +502,12 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The dimension of the encoder output. + """ return self._output_size def forward( @@ -428,18 +518,30 @@ def forward( ctc: CTC = None, max_layer: int = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. + """ + Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - ctc (CTC): Intermediate CTC module. - max_layer (int): Layer depth below which InterCTC is applied. + prev_states (torch.Tensor): Not used in current implementation. Defaults to None. + ctc (CTC): Intermediate CTC module. Defaults to None. + max_layer (int): Maximum number of layers to process. Defaults to None. + Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. + Tuple[Union[torch.Tensor, Tuple[torch.Tensor, List[Tuple[int, torch.Tensor]]]], torch.Tensor, Optional[torch.Tensor]]: + - Output tensor (#batch, L, output_size) or tuple of output tensor and intermediate CTC outputs. + - Output length (#batch). + - None (placeholder for future use). + + Raises: + TooShortUttError: If the input is too short for subsampling. + + Note: + - If intermediate CTC is used, the output will be a tuple containing the final output + and a list of intermediate outputs with their corresponding layer indices. + - The third return value (None) is a placeholder for consistency with other encoders + and may be used in future implementations. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/hubert_encoder.py b/espnet2/asr/encoder/hubert_encoder.py index b2d58c2074e..067294d7f53 100644 --- a/espnet2/asr/encoder/hubert_encoder.py +++ b/espnet2/asr/encoder/hubert_encoder.py @@ -26,63 +26,52 @@ class TorchAudioHuBERTPretrainEncoder(AbsEncoder): - """Torch Audio Hubert encoder module. + """ + TorchAudio Hubert encoder module for pretraining. + + This class implements the Hubert encoder using the TorchAudio implementation. + It can be used for pretraining Hubert models or fine-tuning for downstream tasks. Args: - extractor_mode: Operation mode of feature extractor. - Valid values are "group_norm" or "layer_norm". - extractor_conv_layer_config: Configuration of convolution layers in feature - extractor. List of convolution configuration, - i.e. [[output_channel, kernel_size, stride], ...] - extractor_conv_bias: Whether to include bias term to each convolution - operation. - encoder_embed_dim: The dimension of embedding in encoder. - encoder_projection_dropout: The dropout probability applied after the input - feature is projected to "encoder_embed_dim". - encoder_pos_conv_kernel: Kernel size of convolutional positional embeddings. - encoder_pos_conv_groups: Number of groups of convolutional positional - embeddings. - encoder_num_layers: Number of self attention layers in transformer block. - encoder_num_heads: Number of heads in self attention layers. - encoder_attention_dropout: Dropout probability applied after softmax in - self-attention layer. - encoder_ff_interm_features: Dimension of hidden features in feed forward layer. - encoder_ff_interm_dropout: Dropout probability applied in feedforward layer. - encoder_dropout: Dropout probability applied at the end of feed forward layer. - encoder_layer_norm_first: Control the order of layer norm in transformer layer - and each encoder layer. If True, in transformer layer, layer norm is - applied before features are fed to encoder layers. - encoder_layer_drop: Probability to drop each encoder layer during training. - mask_prob: Probability for each token to be chosen as start of the span - to be masked. - mask_selection: How to choose the mask length. - Options: [static, uniform, normal, poisson]. - mask_other: Secondary mask argument (used for more complex distributions). - mask_length: The lengths of the mask. - no_mask_overlap: Whether to allow masks to overlap. - mask_min_space: Minimum space between spans (if no overlap is enabled). - mask_channel_prob: (float): The probability of replacing a feature with 0. - mask_channel_selection: How to choose the mask length for channel masking. - Options: [static, uniform, normal, poisson]. - mask_channel_other: Secondary mask argument for channel masking(used for more - complex distributions). - mask_channel_length: Minimum space between spans (if no overlap is enabled) - for channel masking. - no_mask_channel_overlap: Whether to allow channel masks to overlap. - mask_channel_min_space: Minimum space between spans for channel - masking(if no overlap is enabled). - skip_masked: If True, skip computing losses over masked frames. - skip_nomask: If True, skip computing losses over unmasked frames. - num_classes: The number of classes in the labels. - final_dim: Project final representations and targets to final_dim. - feature_grad_mult: The factor to scale the convolutional feature extraction - layer gradients by. The scale factor will not affect the forward pass. - finetuning: Whether to finetuning the model with ASR or other tasks. - freeze_encoder_updates: The number of steps to freeze the encoder parameters - in ASR finetuning. - Hubert specific Args: - Please refer to: - https://pytorch.org/audio/stable/generated/torchaudio.models.hubert_pretrain_model.html#torchaudio.models.hubert_pretrain_model + input_size (int, optional): Input feature dimension. Defaults to None. + extractor_mode (str, optional): Operation mode of feature extractor. Defaults to "group_norm". + extractor_conv_layer_config (List[List[int]], optional): Configuration of convolution layers in feature extractor. Defaults to a specific 7-layer configuration. + extractor_conv_bias (bool, optional): Whether to include bias in convolution operations. Defaults to False. + encoder_embed_dim (int, optional): Embedding dimension in encoder. Defaults to 768. + encoder_projection_dropout (float, optional): Dropout probability after input feature projection. Defaults to 0.1. + encoder_pos_conv_kernel (int, optional): Kernel size of convolutional positional embeddings. Defaults to 128. + encoder_pos_conv_groups (int, optional): Number of groups in convolutional positional embeddings. Defaults to 16. + encoder_num_layers (int, optional): Number of self-attention layers in transformer block. Defaults to 12. + encoder_num_heads (int, optional): Number of heads in self-attention layers. Defaults to 12. + encoder_attention_dropout (float, optional): Dropout probability in self-attention layer. Defaults to 0.1. + encoder_ff_interm_features (int, optional): Dimension of hidden features in feed-forward layer. Defaults to 3072. + encoder_ff_interm_dropout (float, optional): Dropout probability in feed-forward layer. Defaults to 0.0. + encoder_dropout (float, optional): Dropout probability at the end of feed-forward layer. Defaults to 0.1. + encoder_layer_norm_first (bool, optional): Controls the order of layer normalization. Defaults to False. + encoder_layer_drop (float, optional): Probability to drop each encoder layer during training. Defaults to 0.05. + mask_prob (float, optional): Probability for each token to be chosen as start of the span to be masked. Defaults to 0.8. + mask_selection (str, optional): How to choose the mask length. Defaults to "static". + mask_other (float, optional): Secondary mask argument. Defaults to 0.0. + mask_length (int, optional): The lengths of the mask. Defaults to 10. + no_mask_overlap (bool, optional): Whether to allow masks to overlap. Defaults to False. + mask_min_space (int, optional): Minimum space between spans if no overlap is enabled. Defaults to 1. + mask_channel_prob (float, optional): Probability of replacing a feature with 0. Defaults to 0.0. + mask_channel_selection (str, optional): How to choose the mask length for channel masking. Defaults to "static". + mask_channel_other (float, optional): Secondary mask argument for channel masking. Defaults to 0.0. + mask_channel_length (int, optional): Minimum space between spans for channel masking. Defaults to 10. + no_mask_channel_overlap (bool, optional): Whether to allow channel masks to overlap. Defaults to False. + mask_channel_min_space (int, optional): Minimum space between spans for channel masking if no overlap is enabled. Defaults to 1. + skip_masked (bool, optional): Whether to skip computing losses over masked frames. Defaults to False. + skip_nomask (bool, optional): Whether to skip computing losses over unmasked frames. Defaults to False. + num_classes (int, optional): The number of classes in the labels. Defaults to 100. + final_dim (int, optional): Project final representations and targets to this dimension. Defaults to 256. + feature_grad_mult (float, optional): The factor to scale gradients from the convolutional feature extraction layer. Defaults to 0.1. + finetuning (bool, optional): Whether to fine-tune the model with ASR or other tasks. Defaults to False. + freeze_encoder_updates (int, optional): The number of steps to freeze the encoder parameters in ASR fine-tuning. Defaults to 0. + + Note: + For more details on the Hubert model and its parameters, refer to the TorchAudio documentation: + https://pytorch.org/audio/stable/generated/torchaudio.models.hubert_pretrain_model.html """ @typechecked @@ -186,6 +175,16 @@ def __init__( self.freeze_encoder_updates = freeze_encoder_updates def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The dimension of the encoder's output. + + Note: + This method returns the value of the `_output_size` attribute, + which is typically set during the initialization of the encoder. + """ return self._output_size def forward( @@ -196,14 +195,32 @@ def forward( ys_pad_length: torch.Tensor = None, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Forward Hubert Pretrain Encoder. + """ + Forward pass of the Hubert Pretrain Encoder. + + This method processes the input tensor through the encoder, applying different + forward passes based on whether the model is in pretraining, fine-tuning, or + evaluation mode. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + ys_pad (torch.Tensor, optional): Target tensor for pretraining. Defaults to None. + ys_pad_length (torch.Tensor, optional): Lengths of target tensor. Defaults to None. + prev_states (torch.Tensor, optional): Previous states. Not used in current implementation. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - Output tensor after forward pass. + - Output lengths. + - Optional additional information (e.g., feature penalty in pretraining). + + Note: + The behavior of this method differs based on the encoder's mode: + - In pretraining mode, it returns logit_m, logit_u, and feature_penalty. + - In fine-tuning mode, it applies masking and returns the encoded features. + - In evaluation mode, it processes the input without masking. """ if not self.finetuning: @@ -276,27 +293,65 @@ def _eval_forward(self, xs_pad, ilens): return x, lengths, None def reload_pretrained_parameters(self): - self.hubert_pretrain_model.load_state_dict(self.pretrained_params, strict=False) + """ + Reload the pretrained parameters of the Hubert model. + + This method reloads the pretrained parameters stored in the `pretrained_params` + attribute into the `hubert_pretrain_model`. It's useful for resetting the model + to its initial pretrained state, especially after fine-tuning or making changes + to the model parameters. + + Note: + This method uses `strict=False` when loading the state dict, which means + it will ignore any parameters in the pretrained state that don't match + the current model structure. This allows for flexibility in model architecture + changes while still loading compatible pretrained weights. + + Raises: + Any exceptions raised by `load_state_dict` method of PyTorch modules. + + Example: + >>> encoder = TorchAudioHuBERTPretrainEncoder() + >>> # After some training or parameter modifications + >>> encoder.reload_pretrained_parameters() + >>> print("Pretrained Hubert model parameters reloaded!") + """ logging.info("Pretrained Hubert model parameters reloaded!") class FairseqHubertEncoder(AbsEncoder): - """FairSeq Hubert encoder module, used for loading pretrained weight and finetuning + """ + FairSeq Hubert encoder module for loading pretrained weights and fine-tuning. + + This class implements the Hubert encoder using the FairSeq implementation. + It supports loading pretrained Hubert models and fine-tuning for downstream tasks. Args: - input_size: input dim - hubert_url: url to Hubert pretrained model - hubert_dir_path: directory to download the Wav2Vec2.0 pretrained model. - output_size: dimension of attention - normalize_before: whether to use layer_norm before the first block - freeze_finetune_updates: steps that freeze all layers except output layer - before tuning the whole model (nessasary to prevent overfit). - dropout_rate: dropout rate - activation_dropout: dropout rate in activation function - attention_dropout: dropout rate in attention - Hubert specific Args: - Please refer to: - https://github.com/pytorch/fairseq/blob/master/fairseq/models/hubert/hubert.py + input_size (int): Input feature dimension. + hubert_url (str, optional): URL to the Hubert pretrained model. Defaults to "./". + hubert_dir_path (str, optional): Directory to download the Hubert pretrained model. Defaults to "./". + output_size (int, optional): Dimension of the encoder output. Defaults to 256. + normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to False. + freeze_finetune_updates (int, optional): Number of updates to freeze all layers except the output layer. Defaults to 0. + dropout_rate (float, optional): Dropout rate. Defaults to 0.0. + activation_dropout (float, optional): Dropout rate in activation function. Defaults to 0.1. + attention_dropout (float, optional): Dropout rate in attention layers. Defaults to 0.0. + mask_length (int, optional): Length of the mask for Hubert. Defaults to 10. + mask_prob (float, optional): Probability of applying mask. Defaults to 0.75. + mask_selection (str, optional): Method of selecting masks. Defaults to "static". + mask_other (int, optional): Secondary mask argument. Defaults to 0. + apply_mask (bool, optional): Whether to apply masking during fine-tuning. Defaults to True. + mask_channel_length (int, optional): Length of the channel mask. Defaults to 64. + mask_channel_prob (float, optional): Probability of applying channel mask. Defaults to 0.5. + mask_channel_other (int, optional): Secondary channel mask argument. Defaults to 0. + mask_channel_selection (str, optional): Method of selecting channel masks. Defaults to "static". + layerdrop (float, optional): Probability of dropping a layer. Defaults to 0.1. + feature_grad_mult (float, optional): Multiplier for feature extractors gradient. Defaults to 0.0. + + Note: + This class requires FairSeq to be installed. It loads the Hubert model + using FairSeq's model loading utilities and supports various masking + and fine-tuning strategies specific to the Hubert architecture. """ @typechecked @@ -432,6 +487,17 @@ def __init__( self.register_buffer("num_updates", torch.LongTensor([0])) def output_size(self) -> int: + """ + Get the output size of the FairSeq Hubert encoder. + + Returns: + int: The dimension of the encoder's output. + + Note: + This method returns the value of the `_output_size` attribute, + which is set during the initialization of the encoder. It represents + the dimension of the output features produced by the encoder. + """ return self._output_size def forward( @@ -440,14 +506,31 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Forward Hubert ASR Encoder. + """ + Forward pass of the FairSeq Hubert ASR Encoder. + + This method processes the input tensor through the Hubert encoder, + applying masking if specified and handling the freezing of parameters + during the initial fine-tuning updates. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - xs_pad (torch.Tensor): Encoded features of shape (B, T, C), where T is the + output sequence length and C is the output dimension. + - olens (torch.Tensor): Output lengths of shape (B,). + - None: Placeholder for future extensions. + + Note: + - The method handles the gradual unfreezing of layers during fine-tuning. + - It applies masking based on the `apply_mask` attribute and the training state. + - The output may be further processed by an additional output layer if specified. + - Layer normalization may be applied before returning the output if `normalize_before` is True. """ masks = make_pad_mask(ilens).to(xs_pad.device) @@ -486,26 +569,60 @@ def forward( return xs_pad, olens, None def reload_pretrained_parameters(self): - self.encoders.load_state_dict(self.pretrained_params, strict=False) + """ + Reload the pretrained parameters of the FairSeq Hubert encoder. + + This method reloads the pretrained parameters stored in the `pretrained_params` + attribute into the `encoders` module of the FairSeq Hubert encoder. It's useful + for resetting the model to its initial pretrained state, especially after + fine-tuning or making changes to the model parameters. + + Note: + This method uses `strict=False` when loading the state dict, which means + it will ignore any parameters in the pretrained state that don't match + the current model structure. This allows for flexibility in model architecture + changes while still loading compatible pretrained weights. + + Raises: + Any exceptions raised by `load_state_dict` method of PyTorch modules. + + Example: + >>> encoder = FairseqHubertEncoder(input_size=80) + >>> # After some training or parameter modifications + >>> encoder.reload_pretrained_parameters() + >>> print("Pretrained Hubert model parameters reloaded!") + """ logging.info("Pretrained Hubert model parameters reloaded!") class FairseqHubertPretrainEncoder(AbsEncoder): - """FairSeq Hubert pretrain encoder module, only used for pretraining stage + """ + FairSeq Hubert pretrain encoder module, specifically designed for the pretraining stage. + + This class implements the Hubert encoder using the FairSeq implementation, tailored + for pretraining tasks. It sets up the Hubert model with specified configurations + and prepares it for pretraining on unlabeled data. Args: - input_size: input dim - output_size: dimension of attention - linear_units: dimension of feedforward layers - attention_heads: the number of heads of multi head attention - num_blocks: the number of encoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - hubert_dict: target dictionary for Hubert pretraining - label_rate: label frame rate. -1 for sequence label - sample_rate: target sample rate. - use_amp: whether to use automatic mixed precision - normalize_before: whether to use layer_norm before the first block + input_size (int, optional): Input feature dimension. Defaults to 1. + output_size (int, optional): Dimension of the encoder output. Defaults to 1024. + linear_units (int, optional): Dimension of feedforward layers. Defaults to 1024. + attention_heads (int, optional): Number of attention heads. Defaults to 12. + num_blocks (int, optional): Number of encoder blocks. Defaults to 12. + dropout_rate (float, optional): Dropout rate. Defaults to 0.0. + attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. + activation_dropout_rate (float, optional): Dropout rate for activation functions. Defaults to 0.0. + hubert_dict (str, optional): Path to the Hubert dictionary file. Defaults to "./dict.txt". + label_rate (int, optional): Label frame rate. Use -1 for sequence labels. Defaults to 100. + checkpoint_activations (bool, optional): Whether to use activation checkpointing. Defaults to False. + sample_rate (int, optional): Audio sample rate. Defaults to 16000. + use_amp (bool, optional): Whether to use automatic mixed precision. Defaults to False. + **kwargs: Additional keyword arguments for Hubert configuration. + + Note: + This class requires FairSeq to be installed. It sets up the Hubert model + using FairSeq's HubertConfig and HubertPretrainingConfig, and prepares + the model for pretraining tasks. """ @typechecked @@ -584,6 +701,17 @@ def _build_dictionary(self, dictionary, hubert_dict_path): self.dictionaries = [dictionary] def output_size(self) -> int: + """ + Get the output size of the FairSeq Hubert pretrain encoder. + + Returns: + int: The dimension of the encoder's output. + + Note: + This method returns the value of the `_output_size` attribute, + which is set during the initialization of the encoder. It represents + the dimension of the output features produced by the Hubert pretrain encoder. + """ return self._output_size def forward( @@ -594,14 +722,28 @@ def forward( ys_pad_length: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Forward Hubert Pretrain Encoder. + """ + Forward pass of the FairSeq Hubert Pretrain Encoder. + + This method processes the input tensor through the Hubert encoder for pretraining, + applying necessary masking and computing the pretraining objectives. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of shape (B,). + ys_pad (torch.Tensor): Target tensor for pretraining tasks. + ys_pad_length (torch.Tensor): Lengths of the target tensor. + prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing the encoder outputs. + The exact content of the tuple depends on the Hubert pretraining configuration. + + Note: + - This method first casts the mask embedding to the appropriate data type. + - It then applies masking to the input and processes it through the Hubert encoder. + - The method is specifically designed for the pretraining phase of Hubert. """ self.cast_mask_emb() masks = make_pad_mask(ilens).to(xs_pad.device) @@ -616,10 +758,48 @@ def forward( return enc_outputs def cast_mask_emb(self): + """ + Cast the mask embedding to half precision if using automatic mixed precision. + + This method checks if automatic mixed precision (AMP) is enabled and if the + mask embedding is not already in half precision. If these conditions are met, + it casts the mask embedding to torch.cuda.HalfTensor. + + Note: + This method is typically called before the forward pass to ensure + that the mask embedding is in the correct precision for AMP computations. + It modifies the `mask_emb` parameter of the encoder in-place. + + Example: + >>> encoder = FairseqHubertPretrainEncoder(use_amp=True) + >>> encoder.cast_mask_emb() + # The mask_emb will be cast to half precision if it wasn't already + """ if self.use_amp and self.encoder.mask_emb.dtype != torch.cuda.HalfTensor: self.encoder.mask_emb = torch.nn.Parameter(self.encoder.mask_emb.half()) def reload_pretrained_parameters(self): + """ + Reinitialize the mask embedding of the Hubert encoder. + + This method reinitializes the mask embedding of the Hubert encoder with + random values drawn from a uniform distribution. The embedding is set to + half precision (float16) and its size is determined by the encoder's + configuration. + + Note: + - This method is typically used to reset the mask embedding to a new + random state, which can be useful when restarting pretraining or + when adapting a pretrained model to a new task. + - The method logs information about the reinitialization, including + the data type of the new mask embedding and whether automatic mixed + precision (AMP) is being used. + + Example: + >>> encoder = FairseqHubertPretrainEncoder() + >>> encoder.reload_pretrained_parameters() + # Logs: "Hubert mask embedding re-initialized!, torch.cuda.HalfTensor, False" + """ self.encoder.mask_emb = torch.nn.Parameter( torch.HalfTensor(self.cfg.encoder_embed_dim).uniform_() ) @@ -631,7 +811,32 @@ def reload_pretrained_parameters(self): def download_hubert(model_url, dir_path): - os.makedirs(dir_path, exist_ok=True) + """ + Download the Hubert model from the given URL and save it to the specified directory. + + This function downloads the Hubert model if it doesn't exist in the specified directory. + It uses a file lock to ensure thread-safe downloading. + + Args: + model_url (str): The URL of the Hubert model to download. + dir_path (str): The directory path where the model should be saved. + + Returns: + str: The full path to the downloaded model file. + + Raises: + Any exceptions raised by os.makedirs, FileLock, or torch.hub.download_url_to_file. + + Example: + >>> model_url = "https://example.com/hubert_model.pt" + >>> dir_path = "./models" + >>> model_path = download_hubert(model_url, dir_path) + >>> print(f"Model downloaded to: {model_path}") + + Note: + This function uses FileLock to prevent multiple processes from downloading + the same model simultaneously. Make sure the FileLock library is installed. + """ model_name = model_url.split("/")[-1] model_path = os.path.join(dir_path, model_name) diff --git a/espnet2/asr/encoder/hugging_face_transformers_encoder.py b/espnet2/asr/encoder/hugging_face_transformers_encoder.py index 1d363764ca3..90bc60ab6e1 100644 --- a/espnet2/asr/encoder/hugging_face_transformers_encoder.py +++ b/espnet2/asr/encoder/hugging_face_transformers_encoder.py @@ -23,7 +23,36 @@ class HuggingFaceTransformersEncoder(AbsEncoder): - """Hugging Face Transformers PostEncoder.""" + """ + Hugging Face Transformers PostEncoder for ASR tasks. + + This class implements an encoder that utilizes pre-trained Hugging Face Transformer + models for automatic speech recognition (ASR) tasks. It can be used as a post-encoder + in ESPnet2 ASR systems. + + Attributes: + transformer (transformers.PreTrainedModel): The Transformer model used for encoding. + pretrained_params (dict): A copy of the initial pre-trained model parameters. + lang_token_id (int): The token ID for language, used if language-specific encoding is needed. + + Args: + input_size (int): The size of the input features. + model_name_or_path (str): The name or path of the pre-trained Hugging Face Transformer model. + lang_token_id (int, optional): The token ID for language. Defaults to -1 (not used). + + Raises: + ImportError: If the 'transformers' library is not installed. + + Note: + This class requires the 'transformers' library to be installed. If not available, + it will raise an ImportError with instructions on how to install it. + + Examples: + >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") + >>> input_tensor = torch.randn(32, 100, 80) # (batch_size, sequence_length, input_size) + >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) + >>> output, output_lengths = encoder(input_tensor, input_lengths) + """ @typechecked def __init__( @@ -56,7 +85,35 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward.""" + """ + Forward pass of the Hugging Face Transformers encoder. + + This method processes the input tensor through the Transformer model and returns + the encoded representation along with the updated input lengths. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, sequence_length, input_size). + input_lengths (torch.Tensor): Tensor of input sequence lengths of shape (batch_size,). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - output (torch.Tensor): Encoded representation from the Transformer model. + Shape: (batch_size, sequence_length, hidden_size). + - input_lengths (torch.Tensor): Updated input lengths, accounting for any + additional tokens (e.g., language token) added during processing. + + Note: + If a language token ID is specified (self.lang_token_id != -1), it will be + prepended to the input sequences, and the input lengths will be incremented accordingly. + + Examples: + >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") + >>> input_tensor = torch.randn(32, 100, 80) # (batch_size, sequence_length, input_size) + >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) + >>> output, output_lengths = encoder.forward(input_tensor, input_lengths) + >>> print(output.shape) # Expected: torch.Size([32, 100, 768]) + >>> print(output_lengths.shape) # Expected: torch.Size([32]) + """ args = {"return_dict": True} @@ -81,11 +138,43 @@ def forward( return output, input_lengths def reload_pretrained_parameters(self): - self.transformer.load_state_dict(self.pretrained_params) + """ + Reload the pretrained parameters of the Transformer model. + + This method resets the Transformer model's parameters to their initial pretrained state. + It's useful for resetting the model to its original configuration after fine-tuning or + other modifications. + + Note: + This method uses the `pretrained_params` attribute, which is a deep copy of the + initial model parameters created during the initialization of the + HuggingFaceTransformersEncoder instance. + + Examples: + >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") + >>> # After some fine-tuning or parameter updates + >>> encoder.reload_pretrained_parameters() + >>> print("Pretrained Transformers model parameters reloaded!") + """ logging.info("Pretrained Transformers model parameters reloaded!") def output_size(self) -> int: - """Get the output size.""" + """ + Get the output size of the Transformer encoder. + + This method returns the size of the hidden state output by the Transformer model. + + Returns: + int: The size of the hidden state (number of features) in the output of the Transformer model. + + Note: + The output size is determined by the `hidden_size` parameter in the Transformer model's configuration. + + Examples: + >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") + >>> output_size = encoder.output_size() + >>> print(output_size) # Expected: 768 for bert-base-uncased + """ return self.transformer.config.hidden_size diff --git a/espnet2/asr/encoder/linear_encoder.py b/espnet2/asr/encoder/linear_encoder.py index 3cf2a607c3c..696871e0f9a 100644 --- a/espnet2/asr/encoder/linear_encoder.py +++ b/espnet2/asr/encoder/linear_encoder.py @@ -22,16 +22,43 @@ class LinearEncoder(AbsEncoder): - """Linear encoder module. + """ + Linear encoder module for processing input sequences. + + This class implements a linear encoder that can be used in various sequence processing tasks, + such as speech recognition or natural language processing. It supports different types of + input layers and can apply normalization and dropout to the input. + + Attributes: + embed: The input embedding layer, which can be one of several types (linear, conv2d, etc.). + normalize_before: A boolean indicating whether to apply layer normalization before processing. + after_norm: A LayerNorm layer applied after processing if normalize_before is True. Args: - input_size: input dim - output_size: dimension of attention - linear_units: the number of units of position-wise feed forward - dropout_rate: dropout rate - input_layer: input layer type - normalize_before: whether to use layer_norm before the first block - padding_idx: padding_idx for input_layer=embed + input_size (int): The dimensionality of the input features. + output_size (int, optional): The dimensionality of the output features. Defaults to 256. + dropout_rate (float, optional): The dropout rate to apply. Defaults to 0.1. + input_layer (str, optional): The type of input layer to use. Can be 'linear', 'conv2d', + 'conv2d2', 'conv2d6', 'conv2d8', 'embed', or None. Defaults to 'conv2d'. + normalize_before (bool, optional): Whether to apply layer normalization before processing. + Defaults to True. + padding_idx (int, optional): The index used for padding in the embedding layer. + Only used when input_layer is 'embed'. Defaults to -1. + + Raises: + ValueError: If an unknown input_layer type is specified. + + Examples: + >>> encoder = LinearEncoder(input_size=80, output_size=256, input_layer='conv2d') + >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) + >>> input_lengths = torch.full((32,), 1000) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 250, 256]) # Time dimension is reduced due to conv2d subsampling + + Note: + The actual behavior of the encoder depends on the chosen input_layer type. + Some input layers (like conv2d variants) may modify the sequence length. """ @typechecked @@ -79,6 +106,17 @@ def __init__( self.after_norm = LayerNorm(output_size) def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The dimensionality of the output features. + + Example: + >>> encoder = LinearEncoder(input_size=80, output_size=256) + >>> print(encoder.output_size()) + 256 + """ return self._output_size def forward( @@ -87,14 +125,40 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Forward pass of the LinearEncoder. + + This method processes the input tensor through the encoder, applying the specified + input layer, normalization, and any other transformations defined in the encoder. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input feature dimension. + ilens (torch.Tensor): Input lengths of each sequence in the batch, shape (B,). + prev_states (torch.Tensor, optional): Not used in this implementation. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - xs_pad (torch.Tensor): Encoded output tensor of shape (B, L', D'), + where L' is the potentially modified sequence length and D' is the output dimension. + - olens (torch.Tensor): Output lengths of each sequence in the batch, shape (B,). + - None: Placeholder for consistency with other encoder implementations. + + Raises: + TooShortUttError: If the input sequence is too short for subsampling operations + when using certain input layers (e.g., Conv2dSubsampling variants). + + Examples: + >>> encoder = LinearEncoder(input_size=80, output_size=256) + >>> xs_pad = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) + >>> ilens = torch.full((32,), 1000) + >>> output, olens, _ = encoder(xs_pad, ilens) + >>> print(output.shape, olens.shape) + torch.Size([32, 1000, 256]) torch.Size([32]) + + Note: + The actual output shape may vary depending on the input_layer type used in the encoder. + Some input layers (like conv2d variants) may reduce the sequence length. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/longformer_encoder.py b/espnet2/asr/encoder/longformer_encoder.py index 1de6c75c555..65c28446447 100644 --- a/espnet2/asr/encoder/longformer_encoder.py +++ b/espnet2/asr/encoder/longformer_encoder.py @@ -35,46 +35,53 @@ class LongformerEncoder(ConformerEncoder): - """Longformer SA Conformer encoder module. + """ + Longformer SA Conformer encoder module. + + This class implements a Longformer Self-Attention Conformer encoder, which extends the + ConformerEncoder to handle longer sequences efficiently using the Longformer attention mechanism. Args: input_size (int): Input dimension. - output_size (int): Dimension of attention. - attention_heads (int): The number of heads of multi head attention. - linear_units (int): The number of units of position-wise feed forward. - num_blocks (int): The number of decoder blocks. - dropout_rate (float): Dropout rate. - attention_dropout_rate (float): Dropout rate in attention. - positional_dropout_rate (float): Dropout rate after adding positional encoding. - input_layer (Union[str, torch.nn.Module]): Input layer type. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - If True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - If False, no additional linear will be applied. i.e. x -> x + att(x) - positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. - rel_pos_type (str): Whether to use the latest relative positional encoding or - the legacy one. The legacy relative positional encoding will be deprecated - in the future. More Details can be found in - https://github.com/espnet/espnet/pull/2816. - encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. - encoder_attn_layer_type (str): Encoder attention layer type. - activation_type (str): Encoder activation function type. - macaron_style (bool): Whether to use macaron style for positionwise layer. - use_cnn_module (bool): Whether to use convolution module. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - cnn_module_kernel (int): Kernerl size of convolution module. - padding_idx (int): Padding idx for input_layer=embed. - attention_windows (list): Layer-wise attention window sizes - for longformer self-attn - attention_dilation(list): Layer-wise attention dilation sizes - for longformer self-attn - attention_mode(str): Implementation for longformer self-attn. - Default="sliding_chunks" - Choose 'n2', 'tvm' or 'sliding_chunks'. More details in - https://github.com/allenai/longformer - + output_size (int): Dimension of attention. Defaults to 256. + attention_heads (int): The number of heads of multi head attention. Defaults to 4. + linear_units (int): The number of units of position-wise feed forward. Defaults to 2048. + num_blocks (int): The number of encoder blocks. Defaults to 6. + dropout_rate (float): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float): Dropout rate after adding positional encoding. Defaults to 0.1. + attention_dropout_rate (float): Dropout rate in attention. Defaults to 0.0. + input_layer (str): Input layer type. Defaults to "conv2d". + normalize_before (bool): Whether to use layer_norm before the first block. Defaults to True. + concat_after (bool): Whether to concat attention layer's input and output. Defaults to False. + positionwise_layer_type (str): Type of positionwise layer. Defaults to "linear". + positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. Defaults to 3. + macaron_style (bool): Whether to use macaron style for positionwise layer. Defaults to False. + rel_pos_type (str): Type of relative positional encoding. Defaults to "legacy". + pos_enc_layer_type (str): Encoder positional encoding layer type. Defaults to "abs_pos". + selfattention_layer_type (str): Encoder attention layer type. Defaults to "lf_selfattn". + activation_type (str): Encoder activation function type. Defaults to "swish". + use_cnn_module (bool): Whether to use convolution module. Defaults to True. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. Defaults to False. + cnn_module_kernel (int): Kernel size of convolution module. Defaults to 31. + padding_idx (int): Padding idx for input_layer=embed. Defaults to -1. + interctc_layer_idx (List[int]): Layer indices for intermediate CTC. Defaults to []. + interctc_use_conditioning (bool): Whether to use intermediate CTC predictions for conditioning. Defaults to False. + attention_windows (list): Layer-wise attention window sizes for longformer self-attn. Defaults to [100, 100, 100, 100, 100, 100]. + attention_dilation (list): Layer-wise attention dilation sizes for longformer self-attn. Defaults to [1, 1, 1, 1, 1, 1]. + attention_mode (str): Implementation for longformer self-attn. Defaults to "sliding_chunks". + + Attributes: + embed (torch.nn.Module): Input embedding layer. + encoders (torch.nn.Module): Encoder layers. + after_norm (torch.nn.Module): Layer normalization applied after the encoder layers. + interctc_layer_idx (List[int]): Layer indices for intermediate CTC. + interctc_use_conditioning (bool): Whether to use intermediate CTC predictions for conditioning. + conditioning_layer (torch.nn.Module): Conditioning layer for intermediate CTC. + + Note: + The Longformer attention mechanism allows for efficient processing of long sequences + by using a combination of local and global attention patterns. This implementation + supports different attention modes, window sizes, and dilation rates for each layer. """ @typechecked @@ -286,6 +293,12 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: + """ + Returns the output size of the encoder. + + Returns: + int: The dimension of the encoder output. + """ return self._output_size def forward( @@ -296,20 +309,30 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. + """ + Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - ctc (CTC): ctc module for intermediate CTC loss - return_all_hs (bool): whether to return all hidden states + prev_states (torch.Tensor, optional): Not used in current implementation. Defaults to None. + ctc (CTC, optional): CTC module for intermediate CTC loss. Defaults to None. + return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - torch.Tensor: Output tensor (#batch, L, output_size). + - torch.Tensor: Output length (#batch). + - torch.Tensor: Not used in current implementation (always None). + + Raises: + TooShortUttError: If the input is too short for subsampling. + + Note: + This method applies the Longformer encoder to the input tensor, handling + padding, subsampling, and intermediate CTC calculations if specified. + It supports returning intermediate outputs for all encoder layers when + return_all_hs is True. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/rnn_encoder.py b/espnet2/asr/encoder/rnn_encoder.py index bc4912b5ec7..439dcafacea 100644 --- a/espnet2/asr/encoder/rnn_encoder.py +++ b/espnet2/asr/encoder/rnn_encoder.py @@ -10,17 +10,45 @@ class RNNEncoder(AbsEncoder): - """RNNEncoder class. + """ + RNN-based encoder for speech recognition tasks. + + This class implements a recurrent neural network (RNN) encoder, which can be + used as part of a speech recognition system. It supports both LSTM and GRU + cell types, with options for bidirectionality and projection layers. + + Attributes: + _output_size (int): The size of the output features. + rnn_type (str): The type of RNN cell used ('lstm' or 'gru'). + bidirectional (bool): Whether the RNN is bidirectional. + use_projection (bool): Whether to use projection layers. + enc (torch.nn.ModuleList): List containing the RNN module(s). Args: - input_size: The number of expected features in the input - output_size: The number of output features - hidden_size: The number of hidden features - bidirectional: If ``True`` becomes a bidirectional LSTM - use_projection: Use projection layer or not - num_layers: Number of recurrent layers - dropout: dropout probability + input_size (int): The number of expected features in the input. + rnn_type (str, optional): The type of RNN cell to use. Defaults to "lstm". + bidirectional (bool, optional): If True, becomes a bidirectional RNN. Defaults to True. + use_projection (bool, optional): Whether to use projection layers. Defaults to True. + num_layers (int, optional): Number of recurrent layers. Defaults to 4. + hidden_size (int, optional): The number of features in the hidden state. Defaults to 320. + output_size (int, optional): The number of output features. Defaults to 320. + dropout (float, optional): Dropout probability. Defaults to 0.0. + subsample (Optional[Sequence[int]], optional): Subsampling factors for each layer. + Defaults to (2, 2, 1, 1). + Raises: + ValueError: If an unsupported rnn_type is provided. + + Example: + >>> input_size = 80 + >>> encoder = RNNEncoder(input_size, rnn_type="lstm", num_layers=3, hidden_size=256) + >>> input_tensor = torch.randn(32, 100, input_size) # (batch_size, sequence_length, input_size) + >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) + >>> print(output.shape) # Expected: torch.Size([32, 25, 320]) + + Note: + The actual output sequence length may be shorter than the input due to subsampling. """ @typechecked @@ -88,6 +116,17 @@ def __init__( ) def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The size of the output features from the encoder. + + Example: + >>> encoder = RNNEncoder(input_size=80, output_size=512) + >>> print(encoder.output_size()) + 512 + """ return self._output_size def forward( @@ -96,6 +135,36 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Forward pass of the RNN encoder. + + This method processes the input tensor through the RNN layers, applying + subsampling and masking as configured. + + Args: + xs_pad (torch.Tensor): Padded input tensor of shape (batch, time, feat). + ilens (torch.Tensor): Input lengths of each sequence in the batch. + prev_states (torch.Tensor, optional): Previous hidden states for incremental processing. + Defaults to None. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + - xs_pad (torch.Tensor): Output tensor after encoding and masking. + - ilens (torch.Tensor): Output lengths of each sequence in the batch. + - current_states (List[torch.Tensor]): List of current hidden states of the RNN. + + Example: + >>> encoder = RNNEncoder(input_size=80, hidden_size=320, output_size=320) + >>> xs_pad = torch.randn(2, 100, 80) # (batch_size, time, feat) + >>> ilens = torch.tensor([100, 80]) + >>> output, out_lens, states = encoder.forward(xs_pad, ilens) + >>> print(output.shape) # Expected: torch.Size([2, 25, 320]) + >>> print(out_lens) # Expected: tensor([25, 20]) + + Note: + The output tensor is masked based on the output lengths to ensure + padded regions are set to zero. + """ if prev_states is None: prev_states = [None] * len(self.enc) assert len(prev_states) == len(self.enc) diff --git a/espnet2/asr/encoder/transformer_encoder.py b/espnet2/asr/encoder/transformer_encoder.py index ca42ede6359..9d9ac0853ea 100644 --- a/espnet2/asr/encoder/transformer_encoder.py +++ b/espnet2/asr/encoder/transformer_encoder.py @@ -36,28 +36,43 @@ class TransformerEncoder(AbsEncoder): - """Transformer encoder module. + """ + Transformer encoder module. + + This class implements a Transformer encoder, which is a key component in sequence-to-sequence models. + It processes input sequences through multiple self-attention and feed-forward layers. Args: - input_size: input dim - output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - positional_dropout_rate: dropout rate after adding positional encoding - input_layer: input layer type - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - positionwise_layer_type: linear of conv1d - positionwise_conv_kernel_size: kernel size of positionwise conv1d layer - padding_idx: padding_idx for input_layer=embed + input_size (int): Dimensionality of the input features. + output_size (int, optional): Dimensionality of the output. Defaults to 256. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in position-wise feed-forward layers. Defaults to 2048. + num_blocks (int, optional): Number of encoder blocks. Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. + attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. + input_layer (str, optional): Type of input layer. Defaults to "conv2d". + pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to apply layer normalization before the first block. Defaults to True. + concat_after (bool, optional): Whether to concatenate attention layer's input and output. Defaults to False. + positionwise_layer_type (str, optional): Type of position-wise layer. Defaults to "linear". + positionwise_conv_kernel_size (int, optional): Kernel size for position-wise conv1d layer. Defaults to 1. + padding_idx (int, optional): Padding index for embedding layer. Defaults to -1. + interctc_layer_idx (List[int], optional): Indices of layers for intermediate CTC. Defaults to []. + interctc_use_conditioning (bool, optional): Whether to use conditioning for intermediate CTC. Defaults to False. + layer_drop_rate (float, optional): Layer dropout rate. Defaults to 0.0. + + Attributes: + embed (torch.nn.Module): Input embedding layer. + encoders (torch.nn.Module): Stack of encoder layers. + after_norm (LayerNorm): Layer normalization applied after the final encoder layer. + interctc_layer_idx (List[int]): Indices of layers for intermediate CTC. + interctc_use_conditioning (bool): Whether to use conditioning for intermediate CTC. + conditioning_layer (torch.nn.Module): Conditioning layer for intermediate CTC. + + Note: + The encoder supports various types of input layers, including linear, convolutional, and embedding layers. + It also allows for intermediate CTC (Connectionist Temporal Classification) computations. """ @typechecked @@ -172,6 +187,16 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: + """ + Returns the output size of the encoder. + + Returns: + int: The dimensionality of the encoder's output. + + Note: + This method returns the value of the `_output_size` attribute, which is set + during the initialization of the TransformerEncoder. + """ return self._output_size def forward( @@ -182,17 +207,40 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Forward pass of the TransformerEncoder. + + This method processes the input through the encoder layers and returns the encoded representation. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - ctc (CTC): ctc module for intermediate CTC loss - return_all_hs (bool): whether to return all hidden states + xs_pad (torch.Tensor): Padded input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of each sequence in the batch. + prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. + ctc (CTC, optional): CTC module for intermediate CTC loss. Defaults to None. + return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - xs_pad (torch.Tensor): Encoded output tensor of shape (B, L', D'), where L' is the + output sequence length and D' is the output dimension. + - olens (torch.Tensor): Output lengths of each sequence in the batch. + - None: Placeholder for future use (e.g., hidden states). + + Raises: + TooShortUttError: If the input sequence is too short for subsampling in the embedding layer. + + Note: + - The method applies input embedding, positional encoding, and multiple transformer encoder layers. + - If intermediate CTC layers are specified, it returns intermediate outputs as well. + - The output can be normalized if `normalize_before` is set to True during initialization. + + Examples: + >>> encoder = TransformerEncoder(input_size=80, output_size=256) + >>> xs_pad = torch.randn(2, 100, 80) # (batch_size, sequence_length, input_dim) + >>> ilens = torch.tensor([100, 80]) # Actual lengths of sequences in the batch + >>> output, olens, _ = encoder(xs_pad, ilens) + >>> print(output.shape) # Example output shape: torch.Size([2, 100, 256]) """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/transformer_encoder_multispkr.py b/espnet2/asr/encoder/transformer_encoder_multispkr.py index 8f79389a822..cabcac823b0 100644 --- a/espnet2/asr/encoder/transformer_encoder_multispkr.py +++ b/espnet2/asr/encoder/transformer_encoder_multispkr.py @@ -33,30 +33,59 @@ class TransformerEncoder(AbsEncoder): - """Transformer encoder module. + """ + Transformer encoder module for speech recognition tasks. + + This class implements a Transformer-based encoder that can be used in various + speech recognition models. It supports different input layer types, positional + encodings, and customizable encoder architectures. + + Attributes: + _output_size (int): The output dimension of the encoder. + embed (torch.nn.Module): The input embedding layer. + normalize_before (bool): Whether to apply layer normalization before each block. + encoders (torch.nn.Module): The main encoder layers. + after_norm (torch.nn.Module): The final layer normalization (if normalize_before is True). + num_inf (int): The number of inference outputs. + encoders_sd (torch.nn.ModuleList): Speaker-dependent encoder layers. Args: - input_size: input dim - output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of recognition encoder blocks - num_blocks_sd: the number of speaker dependent encoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - positional_dropout_rate: dropout rate after adding positional encoding - input_layer: input layer type - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - positionwise_layer_type: linear of conv1d - positionwise_conv_kernel_size: kernel size of positionwise conv1d layer - padding_idx: padding_idx for input_layer=embed - num_inf: number of inference output + input_size (int): Dimension of the input features. + output_size (int, optional): Dimension of the encoder output and attention. + Defaults to 256. + attention_heads (int, optional): Number of attention heads. Defaults to 4. + linear_units (int, optional): Number of units in position-wise feed-forward layers. + Defaults to 2048. + num_blocks (int, optional): Number of encoder blocks. Defaults to 6. + num_blocks_sd (int, optional): Number of speaker-dependent encoder blocks. + Defaults to 6. + dropout_rate (float, optional): Dropout rate. Defaults to 0.1. + positional_dropout_rate (float, optional): Dropout rate for positional encoding. + Defaults to 0.1. + attention_dropout_rate (float, optional): Dropout rate in attention layers. + Defaults to 0.0. + input_layer (str, optional): Type of input layer. Can be "conv2d", "linear", + "conv2d1", "conv2d2", "conv2d6", "conv2d8", "embed", or None. Defaults to "conv2d". + pos_enc_class (type, optional): Positional encoding class. + Defaults to PositionalEncoding. + normalize_before (bool, optional): Whether to use layer normalization before + each block. Defaults to True. + concat_after (bool, optional): Whether to concatenate attention layer's input + and output. Defaults to False. + positionwise_layer_type (str, optional): Type of position-wise layer. + Can be "linear", "conv1d", or "conv1d-linear". Defaults to "linear". + positionwise_conv_kernel_size (int, optional): Kernel size of position-wise + conv1d layer. Defaults to 1. + padding_idx (int, optional): Padding index for input_layer="embed". Defaults to -1. + num_inf (int, optional): Number of inference outputs. Defaults to 1. + + Raises: + ValueError: If an unknown input_layer type is specified. + NotImplementedError: If an unsupported positionwise_layer_type is specified. + + Note: + This implementation is based on the paper "Attention Is All You Need" + by Vaswani et al. (2017) and adapted for speech recognition tasks. """ @typechecked @@ -175,6 +204,12 @@ def __init__( ) def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The output dimension of the encoder. + """ return self._output_size def forward( @@ -183,14 +218,35 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. + """ + Forward pass of the Transformer encoder. + + This method processes the input tensor through the encoder layers, applying + positional encoding, attention mechanisms, and feed-forward networks. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Padded input tensor of shape (B, L, D), where B is + the batch size, L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of each sequence in the batch, shape (B,). + prev_states (torch.Tensor, optional): Not used in the current implementation. + Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - torch.Tensor: Encoded output tensor of shape (B, num_inf, L', D'), + where L' is the encoded sequence length and D' is the output dimension. + - torch.Tensor: Output lengths of each sequence in the batch after + encoding, shape (B, num_inf). + - None: Placeholder for future use, currently always None. + + Raises: + TooShortUttError: If the input sequence is too short for subsampling in + certain input layer types (Conv2dSubsampling variants). + + Note: + The method handles different input layer types and applies the appropriate + embedding and subsampling techniques before passing the data through the + encoder layers. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/vgg_rnn_encoder.py b/espnet2/asr/encoder/vgg_rnn_encoder.py index fd457e7f8ff..28cb27f3db2 100644 --- a/espnet2/asr/encoder/vgg_rnn_encoder.py +++ b/espnet2/asr/encoder/vgg_rnn_encoder.py @@ -11,17 +11,39 @@ class VGGRNNEncoder(AbsEncoder): - """VGGRNNEncoder class. + """ + VGGRNNEncoder class for feature extraction in speech recognition tasks. + + This class combines a VGG (Visual Geometry Group) network with a Recurrent Neural Network (RNN) + to process input features for Automatic Speech Recognition (ASR) tasks. + + Attributes: + _output_size (int): The size of the output features. + rnn_type (str): The type of RNN used ('lstm' or 'gru'). + bidirectional (bool): Whether the RNN is bidirectional. + use_projection (bool): Whether to use projection layer after RNN. + enc (torch.nn.ModuleList): List of encoder modules (VGG2L and RNN/RNNP). Args: - input_size: The number of expected features in the input - bidirectional: If ``True`` becomes a bidirectional LSTM - use_projection: Use projection layer or not - num_layers: Number of recurrent layers - hidden_size: The number of hidden features - output_size: The number of output features - dropout: dropout probability + input_size (int): The number of expected features in the input. + rnn_type (str, optional): Type of RNN to use. Defaults to "lstm". + bidirectional (bool, optional): If True, use bidirectional RNN. Defaults to True. + use_projection (bool, optional): Whether to use projection layer. Defaults to True. + num_layers (int, optional): Number of recurrent layers. Defaults to 4. + hidden_size (int, optional): The number of features in the hidden state. Defaults to 320. + output_size (int, optional): The number of output features. Defaults to 320. + dropout (float, optional): Dropout probability. Defaults to 0.0. + in_channel (int, optional): Number of input channels. Defaults to 1. + Raises: + ValueError: If rnn_type is not 'lstm' or 'gru'. + + Example: + >>> input_size = 80 + >>> encoder = VGGRNNEncoder(input_size, rnn_type="lstm", num_layers=3) + >>> input_tensor = torch.randn(32, 100, input_size) # (batch, time, features) + >>> input_lengths = torch.randint(50, 100, (32,)) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) """ @typechecked @@ -80,6 +102,17 @@ def __init__( ) def output_size(self) -> int: + """ + Returns the output size of the encoder. + + Returns: + int: The number of output features from the encoder. + + Example: + >>> encoder = VGGRNNEncoder(input_size=80, output_size=320) + >>> print(encoder.output_size()) + 320 + """ return self._output_size def forward( @@ -88,6 +121,33 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Forward pass of the VGGRNNEncoder. + + This method processes the input through the VGG layers followed by RNN layers. + + Args: + xs_pad (torch.Tensor): Padded input tensor of shape (batch, time, feat). + ilens (torch.Tensor): Input lengths of each sequence in batch. + prev_states (torch.Tensor, optional): Previous states for RNN layers. Defaults to None. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + - xs_pad (torch.Tensor): Output tensor after encoding. + - ilens (torch.Tensor): Output lengths of each sequence in batch. + - current_states (List[torch.Tensor]): List of current states for RNN layers. + + Example: + >>> encoder = VGGRNNEncoder(input_size=80, output_size=320) + >>> xs_pad = torch.randn(32, 100, 80) # (batch, time, feat) + >>> ilens = torch.randint(50, 100, (32,)) + >>> output, output_lengths, states = encoder(xs_pad, ilens) + >>> print(output.shape) + torch.Size([32, 25, 320]) # (batch, reduced_time, output_size) + + Note: + The time dimension is typically reduced due to the VGG layers' stride operations. + """ if prev_states is None: prev_states = [None] * len(self.enc) assert len(prev_states) == len(self.enc) diff --git a/espnet2/asr/encoder/wav2vec2_encoder.py b/espnet2/asr/encoder/wav2vec2_encoder.py index 8eb535dee3f..50711eba97b 100644 --- a/espnet2/asr/encoder/wav2vec2_encoder.py +++ b/espnet2/asr/encoder/wav2vec2_encoder.py @@ -18,16 +18,38 @@ class FairSeqWav2Vec2Encoder(AbsEncoder): - """FairSeq Wav2Vec2 encoder module. + """ + FairSeq Wav2Vec2 encoder module. + + This class implements an encoder based on the Wav2Vec2.0 model from FairSeq. + It can load pre-trained Wav2Vec2.0 models and use them as feature extractors + for speech recognition tasks. The encoder allows for fine-tuning of the + Wav2Vec2.0 model and provides options for output dimensionality adjustment. + + Attributes: + encoders (fairseq.models.wav2vec.wav2vec2.Wav2Vec2Model): The Wav2Vec2 model. + output_layer (torch.nn.Sequential): Optional layer for output dimension adjustment. + after_norm (espnet.nets.pytorch_backend.transformer.layer_norm.LayerNorm): Optional layer normalization. Args: - input_size: input dim - output_size: dimension of attention - w2v_url: url to Wav2Vec2.0 pretrained model - w2v_dir_path: directory to download the Wav2Vec2.0 pretrained model. - normalize_before: whether to use layer_norm before the first block - finetune_last_n_layers: last n layers to be finetuned in Wav2Vec2.0 - 0 means to finetune every layer if freeze_w2v=False. + input_size (int): Input dimension (not used in the current implementation). + w2v_url (str): URL to the Wav2Vec2.0 pretrained model. + w2v_dir_path (str, optional): Directory to download the Wav2Vec2.0 pretrained model. Defaults to "./". + output_size (int, optional): Dimension of the output. Defaults to 256. + normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to False. + freeze_finetune_updates (int, optional): Number of updates to freeze the model before fine-tuning. Defaults to 0. + + Note: + This class requires FairSeq to be installed. If not installed, it will raise an ImportError + with instructions on how to install FairSeq. + + Examples: + >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") + >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) + >>> input_lengths = torch.full((32,), 1000) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 1000, 256]) """ @typechecked @@ -93,6 +115,21 @@ def __init__( self.register_buffer("num_updates", torch.LongTensor([0])) def output_size(self) -> int: + """ + Get the output size of the encoder. + + Returns: + int: The size of the output tensor along the feature dimension. + + Note: + This method returns the value of the `_output_size` attribute, + which is set during the initialization of the encoder. + + Examples: + >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt", output_size=512) + >>> print(encoder.output_size()) + 512 + """ return self._output_size def forward( @@ -101,14 +138,38 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Forward FairSeqWav2Vec2 Encoder. + """ + Forward pass of the FairSeqWav2Vec2 Encoder. + + This method processes the input tensor through the Wav2Vec2 model and optional output layer. Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. + xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the input dimension. + ilens (torch.Tensor): Input lengths of shape (B,) representing the valid length + of each sequence in the batch. + prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. + Returns: - position embedded tensor and mask + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - torch.Tensor: Output tensor of shape (B, T, C), where T is the output sequence length + and C is the output dimension. + - torch.Tensor: Output lengths of shape (B,) representing the valid length of each + output sequence in the batch. + - Optional[torch.Tensor]: Always None in the current implementation. + + Note: + - The method handles the freezing and unfreezing of parameters based on the + `freeze_finetune_updates` attribute. + - If `normalize_before` is True, layer normalization is applied to the output. + + Examples: + >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") + >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) + >>> input_lengths = torch.full((32,), 1000) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) + >>> print(output.shape, output_lengths.shape) + torch.Size([32, 1000, 256]) torch.Size([32]) """ masks = make_pad_mask(ilens).to(xs_pad.device) @@ -144,12 +205,56 @@ def forward( return xs_pad, olens, None def reload_pretrained_parameters(self): - self.encoders.load_state_dict(self.pretrained_params) + """ + Reload the pretrained parameters of the Wav2Vec model. + + This method resets the encoder's parameters to their original pretrained values. + It's useful when you want to start from the initial pretrained state after + fine-tuning or other modifications to the model. + + Note: + This method uses the `pretrained_params` attribute, which is a deep copy + of the initial model state dictionary created during the encoder's initialization. + + Examples: + >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") + >>> # After some fine-tuning or parameter updates + >>> encoder.reload_pretrained_parameters() + >>> print("Pretrained Wav2Vec model parameters reloaded!") + """ logging.info("Pretrained Wav2Vec model parameters reloaded!") def download_w2v(model_url, dir_path): - os.makedirs(dir_path, exist_ok=True) + """ + Download the Wav2Vec model and its dictionary. + + This function downloads the Wav2Vec model and its corresponding dictionary + if they don't already exist in the specified directory. It uses FileLock + to ensure thread-safe downloads. + + Args: + model_url (str): The URL of the Wav2Vec model to download. + dir_path (str): The directory path where the model and dictionary + should be saved. + + Returns: + str: The local file path of the downloaded Wav2Vec model. + + Raises: + OSError: If there are issues creating the directory or downloading files. + + Examples: + >>> model_url = "https://example.com/wav2vec_model.pt" + >>> dir_path = "./models" + >>> local_model_path = download_w2v(model_url, dir_path) + >>> print(local_model_path) + './models/wav2vec_model.pt' + + Note: + This function also downloads a dictionary file from a hardcoded URL: + 'https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt' + """ model_name = model_url.split("/")[-1] model_path = os.path.join(dir_path, model_name) diff --git a/espnet2/asr/encoder/whisper_encoder.py b/espnet2/asr/encoder/whisper_encoder.py index 5e96b9b8900..f8c3e806d2f 100644 --- a/espnet2/asr/encoder/whisper_encoder.py +++ b/espnet2/asr/encoder/whisper_encoder.py @@ -10,9 +10,46 @@ class OpenAIWhisperEncoder(AbsEncoder): - """Transformer-based Speech Encoder from OpenAI's Whisper Model: - - URL: https://github.com/openai/whisper + """ + OpenAI Whisper-based Speech Encoder for ASR tasks. + + This class implements a speech encoder based on OpenAI's Whisper model, + designed for Automatic Speech Recognition (ASR) tasks. It uses a Transformer + architecture and can be initialized with various Whisper model sizes. + + Attributes: + n_fft (int): Number of FFT points. + win_length (int): Window length for STFT. + hop_length (int): Hop length for STFT. + n_mels (int): Number of mel filterbanks. + mel_filters (function): Function to create mel filterbanks. + dropout (torch.nn.Dropout): Dropout layer. + encoders (whisper.model.Encoder): Whisper encoder layers. + specaug (SpecAug): SpecAugment layer for data augmentation. + do_pad_trim (bool): Whether to pad or trim input audio. + pad_samples (int): Number of samples to pad to. + + Args: + input_size (int): Input feature size. Defaults to 1. + dropout_rate (float): Dropout rate. Defaults to 0.0. + whisper_model (str): Whisper model size to use. Defaults to "small". + download_dir (Optional[str]): Directory to download Whisper model. Defaults to None. + use_specaug (bool): Whether to use SpecAugment. Defaults to False. + specaug_conf (Union[dict, None]): SpecAugment configuration. Defaults to None. + do_pad_trim (bool): Whether to pad or trim input audio. Defaults to False. + + Raises: + ImportError: If the whisper package is not properly installed. + + Note: + This encoder requires the `whisper` package to be installed. + It can be installed using the provided installation script. + + Example: + >>> encoder = OpenAIWhisperEncoder(whisper_model="base", use_specaug=True) + >>> input_tensor = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) """ @typechecked @@ -67,6 +104,21 @@ def __init__( self.pad_samples = N_SAMPLES def output_size(self) -> int: + """ + Returns the output size of the encoder. + + This method returns the dimensionality of the encoder's output features. + + Returns: + int: The size of the output feature vector, which corresponds to the + number of units in the final layer normalization of the encoder. + + Example: + >>> encoder = OpenAIWhisperEncoder(whisper_model="base") + >>> output_dim = encoder.output_size() + >>> print(output_dim) + 512 # This may vary depending on the Whisper model size + """ return self.encoders.ln_post.normalized_shape[-1] def pad_or_trim( @@ -75,9 +127,33 @@ def pad_or_trim( length: int, axis: int = -1, ) -> torch.Tensor: - """Pad or trim the audio array to N_SAMPLES. + """ + Pad or trim the input tensor to a specified length along a given axis. + + This method is used to ensure that the input tensor has a consistent length, + which is particularly useful for zero-shot inference cases. + + Args: + array (torch.Tensor): The input tensor to be padded or trimmed. + length (int): The desired length of the tensor along the specified axis. + axis (int, optional): The axis along which to pad or trim. Defaults to -1 (last dimension). - Used in zero-shot inference cases. + Returns: + torch.Tensor: The padded or trimmed tensor with the specified length along the given axis. + + Raises: + ValueError: If the input tensor has fewer dimensions than the specified axis. + + Example: + >>> encoder = OpenAIWhisperEncoder() + >>> input_tensor = torch.randn(1, 12000) + >>> padded_tensor = encoder.pad_or_trim(input_tensor, length=16000) + >>> print(padded_tensor.shape) + torch.Size([1, 16000]) + + Note: + If the input tensor is longer than the specified length, it will be trimmed. + If it's shorter, it will be padded with zeros. """ if array.shape[axis] > length: array = array.index_select( @@ -96,7 +172,36 @@ def log_mel_spectrogram( audio: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: - """Use log-mel spectrogram computation native to Whisper training""" + """ + Compute the log-mel spectrogram of the input audio. + + This method implements the log-mel spectrogram computation native to Whisper training. + It performs short-time Fourier transform (STFT) on the input audio, applies mel filters, + and computes the log of the resulting spectrogram. + + Args: + audio (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). + ilens (torch.Tensor, optional): Tensor containing the lengths of each audio in the batch. + Defaults to None. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - log_spec (torch.Tensor): Log-mel spectrogram of shape (batch_size, n_mels, time). + - olens (Optional[torch.Tensor]): Tensor containing the lengths of each spectrogram + in the batch. Returns None if ilens is None. + + Note: + - The method uses the Whisper-specific parameters for STFT and mel filterbank. + - The last frame of the STFT is discarded to match Whisper's behavior. + - The log spectrogram is clamped and normalized as per Whisper's preprocessing. + + Example: + >>> encoder = OpenAIWhisperEncoder() + >>> audio = torch.randn(2, 16000) # 2 audio samples of 1 second each at 16kHz + >>> log_spec, spec_lengths = encoder.log_mel_spectrogram(audio) + >>> print(log_spec.shape) + torch.Size([2, 80, 100]) # (batch_size, n_mels, time) + """ window = torch.hann_window(self.win_length).to(audio.device) stft = torch.stft( audio, self.n_fft, self.hop_length, window=window, return_complex=True @@ -128,6 +233,37 @@ def whisper_encode( input: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: + """ + Encode input features using the Whisper encoder. + + This method applies the Whisper encoder to the input features, including + convolutional layers, positional embedding, and transformer blocks. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, n_mels, time). + ilens (torch.Tensor, optional): Tensor containing the lengths of each input + in the batch. Defaults to None. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - x (torch.Tensor): Encoded output of shape (batch_size, time, d_model). + - olens (Optional[torch.Tensor]): Tensor containing the lengths of each + encoded output in the batch. Returns None if ilens is None. + + Note: + - The method applies two convolutional layers followed by transformer blocks. + - Positional embedding is added to the output of convolutional layers. + - Due to positional encoding limitations, audios longer than 30 seconds + may not be fully encoded. + - Dropout is applied between transformer blocks during training. + + Example: + >>> encoder = OpenAIWhisperEncoder() + >>> input_features = torch.randn(2, 80, 100) # (batch_size, n_mels, time) + >>> encoded_output, output_lengths = encoder.whisper_encode(input_features) + >>> print(encoded_output.shape) + torch.Size([2, 100, 512]) # (batch_size, time, d_model) + """ x = F.gelu(self.encoders.conv1(input)) x = F.gelu(self.encoders.conv2(x)) x = x.permute(0, 2, 1) @@ -171,6 +307,39 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Forward pass of the OpenAI Whisper Encoder. + + This method processes the input audio through the entire encoder pipeline, + including optional padding/trimming, log-mel spectrogram computation, + optional SpecAugment, and Whisper encoding. + + Args: + xs_pad (torch.Tensor): Padded input tensor of shape (batch_size, T). + ilens (torch.Tensor): Tensor of input lengths of shape (batch_size,). + prev_states (torch.Tensor, optional): Tensor containing previous states. + Not used in this implementation. Defaults to None. + + Returns: + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + - xs_pad (torch.Tensor): Encoded output of shape (batch_size, T', D). + - olens (torch.Tensor): Tensor of output lengths of shape (batch_size,). + - None: Placeholder for consistency with AbsEncoder interface. + + Note: + - If `do_pad_trim` is True, input will be padded or trimmed to `pad_samples`. + - SpecAugment is applied during training if `specaug` is not None. + - The method handles the entire encoding process from raw audio to + final encoded representations. + + Example: + >>> encoder = OpenAIWhisperEncoder(do_pad_trim=True, use_specaug=True) + >>> input_audio = torch.randn(2, 16000) # 2 audio samples of 1 second each at 16kHz + >>> input_lengths = torch.tensor([16000, 16000]) + >>> output, output_lengths, _ = encoder(input_audio, input_lengths) + >>> print(output.shape) + torch.Size([2, 100, 512]) # (batch_size, time, d_model) + """ if self.do_pad_trim: xs_pad = self.pad_or_trim(xs_pad, self.pad_samples) diff --git a/espnet2/asr/espnet_model.py b/espnet2/asr/espnet_model.py index b5224585cf7..dd2a7c30dee 100644 --- a/espnet2/asr/espnet_model.py +++ b/espnet2/asr/espnet_model.py @@ -35,7 +35,35 @@ def autocast(enabled=True): class ESPnetASRModel(AbsESPnetModel): - """CTC-attention hybrid Encoder-Decoder model""" + """ + CTC-attention hybrid Encoder-Decoder model for Automatic Speech Recognition (ASR). + + This class implements a hybrid model that combines CTC (Connectionist Temporal Classification) + and attention-based approaches for ASR. It supports both regular attention decoder and + transducer decoder architectures. + + The model consists of several components: + - Frontend processing (optional) + - SpecAugment for data augmentation (optional) + - Normalization (optional) + - Pre-encoder (optional) + - Encoder + - Post-encoder (optional) + - Decoder (attention-based or transducer-based) + - CTC module + - Joint network (for transducer architecture) + + It can be used for various ASR tasks and supports features such as: + - CTC/Attention multi-task learning + - Intermediate CTC + - Transducer decoding + - Multi-blank transducer + - Auxiliary CTC tasks + - Multilingual ASR with language token + + The model calculates losses for CTC, attention, and/or transducer components, + and provides error rates (CER/WER) during evaluation. + """ @typechecked def __init__( @@ -209,14 +237,33 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss + """ + Frontend + Encoder + Decoder + Calc loss Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - kwargs: "utt_id" is among the input. + speech (torch.Tensor): Input speech tensor (Batch, Length, ...). + speech_lengths (torch.Tensor): Speech length tensor (Batch, ). + text (torch.Tensor): Text tensor (Batch, Length). + text_lengths (torch.Tensor): Text length tensor (Batch,). + kwargs: Additional keyword arguments. "utt_id" is among the input. + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + - Loss tensor + - Dictionary containing various statistics and metrics: + - loss_ctc: CTC loss (if applicable) + - loss_att: Attention loss (if applicable) + - loss_transducer: Transducer loss (if applicable) + - acc: Attention accuracy + - cer: Character Error Rate + - wer: Word Error Rate + - loss: Total loss + - Batch size as a weight tensor + + Note: + The method expects input tensors to have consistent batch sizes across all inputs. + It supports CTC, attention, and transducer-based loss calculations depending on the model configuration. + Intermediate CTC loss is also calculated if the interctc_weight is non-zero. """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -361,17 +408,56 @@ def collect_feats( text_lengths: torch.Tensor, **kwargs, ) -> Dict[str, torch.Tensor]: + """ + Collect features for the input speech. + + This method extracts features from the input speech using the model's feature extractor. + + Args: + speech (torch.Tensor): Input speech tensor (Batch, Length, ...). + speech_lengths (torch.Tensor): Speech length tensor (Batch, ). + text (torch.Tensor): Text tensor (Batch, Length). + text_lengths (torch.Tensor): Text length tensor (Batch,). + kwargs: Additional keyword arguments. + + Returns: + Dict[str, torch.Tensor]: A dictionary containing: + - "feats": Extracted features tensor. + - "feats_lengths": Lengths of the extracted features. + + Note: + This method is typically used for feature extraction during data preparation or analysis. + It does not perform any encoding or decoding operations. + """ feats, feats_lengths = self._extract_feats(speech, speech_lengths) return {"feats": feats, "feats_lengths": feats_lengths} def encode( self, speech: torch.Tensor, speech_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py + """ + Encode speech features into a latent representation. + + This method performs the following steps: + 1. Extract features from the input speech + 2. Apply data augmentation (if SpecAugment is enabled and in training mode) + 3. Normalize the features (if normalization is enabled) + 4. Apply pre-encoding (if a pre-encoder is defined) + 5. Encode the features using the main encoder + 6. Apply post-encoding (if a post-encoder is defined) Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) + speech (torch.Tensor): Input speech tensor (Batch, Length, ...). + speech_lengths (torch.Tensor): Speech length tensor (Batch, ). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - Encoded speech tensor (Batch, Length2, Dim2). + - Encoded speech lengths tensor (Batch, ). + + Note: + This method is used by asr_inference.py and is a key component of the ASR pipeline. + It handles the entire encoding process from raw input to encoder output. """ with autocast(False): # 1. Extract feats @@ -455,15 +541,25 @@ def nll( ys_pad: torch.Tensor, ys_pad_lens: torch.Tensor, ) -> torch.Tensor: - """Compute negative log likelihood(nll) from transformer-decoder + """ + Compute negative log likelihood (nll) from transformer-decoder. - Normally, this function is called in batchify_nll. + This method calculates the negative log likelihood for the given encoder output + and target sequences using the transformer decoder. Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) + encoder_out (torch.Tensor): Encoder output tensor (Batch, Length, Dim). + encoder_out_lens (torch.Tensor): Encoder output length tensor (Batch,). + ys_pad (torch.Tensor): Padded target sequence tensor (Batch, Length). + ys_pad_lens (torch.Tensor): Target sequence length tensor (Batch,). + + Returns: + torch.Tensor: Negative log likelihood tensor (Batch,). + + Note: + This function is typically called within the batchify_nll method. + It adds start-of-sequence (sos) and end-of-sequence (eos) tokens to the target sequence, + forwards the input through the decoder, and computes the cross-entropy loss. """ ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) ys_in_lens = ys_pad_lens + 1 @@ -494,18 +590,27 @@ def batchify_nll( ys_pad_lens: torch.Tensor, batch_size: int = 100, ): - """Compute negative log likelihood(nll) from transformer-decoder + """ + Compute negative log likelihood (nll) from transformer-decoder in batches. + + This method calculates the negative log likelihood for the given encoder output + and target sequences using the transformer decoder, processing the input in batches + to avoid potential out-of-memory (OOM) issues. - To avoid OOM, this fuction seperate the input into batches. - Then call nll for each batch and combine and return results. Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) - batch_size: int, samples each batch contain when computing nll, - you may change this to avoid OOM or increase - GPU memory usage + encoder_out (torch.Tensor): Encoder output tensor (Batch, Length, Dim). + encoder_out_lens (torch.Tensor): Encoder output length tensor (Batch,). + ys_pad (torch.Tensor): Padded target sequence tensor (Batch, Length). + ys_pad_lens (torch.Tensor): Target sequence length tensor (Batch,). + batch_size (int, optional): Number of samples to process in each batch. Defaults to 100. + + Returns: + torch.Tensor: Negative log likelihood tensor (Batch,). + + Note: + This method is designed to handle large inputs by splitting them into smaller batches. + It calls the nll method for each batch and combines the results. + Adjust the batch_size parameter to optimize memory usage and processing speed. """ total_num = encoder_out.size(0) if total_num <= batch_size: diff --git a/espnet2/asr/frontend/abs_frontend.py b/espnet2/asr/frontend/abs_frontend.py index 8f785e38d9e..5fa4c63504d 100644 --- a/espnet2/asr/frontend/abs_frontend.py +++ b/espnet2/asr/frontend/abs_frontend.py @@ -5,12 +5,86 @@ class AbsFrontend(torch.nn.Module, ABC): + """ + Abstract base class for frontend modules in a neural network. + + This class defines the interface for frontend modules, which are typically + used in speech processing tasks to convert raw input signals into + feature representations. + + Attributes: + None + + Examples: + >>> class MyFrontend(AbsFrontend): + ... def output_size(self) -> int: + ... return 80 + ... def forward(self, input: torch.Tensor, input_lengths: torch.Tensor): + ... # Implementation here + ... return features, feature_lengths + + Note: + Subclasses must implement the `output_size` and `forward` methods. + """ + @abstractmethod def output_size(self) -> int: + """ + Returns the output size of the frontend module. + + This method should be implemented by subclasses to specify the + dimensionality of the feature representation produced by the frontend. + + Returns: + int: The size of the output feature dimension. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Examples: + >>> class MyFrontend(AbsFrontend): + ... def output_size(self) -> int: + ... return 80 + >>> frontend = MyFrontend() + >>> print(frontend.output_size()) + 80 + """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Processes the input and returns the feature representation. + + This method should be implemented by subclasses to define how the input + is transformed into features. + + Args: + input (torch.Tensor): The input tensor to be processed. + input_lengths (torch.Tensor): The lengths of each sequence in the input batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - features (torch.Tensor): The processed features. + - feature_lengths (torch.Tensor): The lengths of each sequence in the feature batch. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Examples: + >>> class MyFrontend(AbsFrontend): + ... def forward(self, input: torch.Tensor, input_lengths: torch.Tensor): + ... # Assume some processing here + ... features = input * 2 + ... feature_lengths = input_lengths + ... return features, feature_lengths + >>> frontend = MyFrontend() + >>> input_tensor = torch.randn(32, 1000) # Batch of 32, sequence length 1000 + >>> input_lengths = torch.full((32,), 1000) + >>> features, feature_lengths = frontend(input_tensor, input_lengths) + >>> features.shape + torch.Size([32, 1000]) + """ raise NotImplementedError diff --git a/espnet2/asr/frontend/asteroid_frontend.py b/espnet2/asr/frontend/asteroid_frontend.py index 4e9237081ae..cab504e964a 100644 --- a/espnet2/asr/frontend/asteroid_frontend.py +++ b/espnet2/asr/frontend/asteroid_frontend.py @@ -16,18 +16,38 @@ class AsteroidFrontend(AbsFrontend): - """Asteroid Filterbank Frontend. - - Provides a Sinc-convolutional-based audio feature extractor. The same - function can be achieved by using `sliding_winodw frontend + - sinc preencoder`. - - NOTE(jiatong): this function is used in sentence-level classification - tasks (e.g., spk). Other usages are not fully investigated. - - NOTE(jeeweon): this function implements the parameterized analytic - filterbank layer in M. Pariente, S. Cornell, A. Deleforge and E. Vincent, - "Filterbank design for end-to-end speech separation," in Proc. ICASSP, 2020 + """ + Asteroid Filterbank Frontend for audio feature extraction. + + This class implements a Sinc-convolutional-based audio feature extractor using the + Asteroid filterbank. It provides functionality similar to using a sliding window + frontend with a sinc preencoder. + + The frontend applies preemphasis, normalization, and frame-wise feature extraction + using parameterized analytic filterbanks as described in Pariente et al. (2020). + + Attributes: + sinc_filters (int): Number of filters for the sinc convolution. + sinc_kernel_size (int): Kernel size for the sinc convolution. + sinc_stride (int): Stride size for the first sinc-conv layer, determining + the compression rate (Hz). + output_dim (int): Output dimension of the feature extraction. + + Note: + This frontend is primarily used in sentence-level classification tasks + (e.g., speaker recognition). Its effectiveness in other applications + has not been fully investigated. + + Example: + >>> frontend = AsteroidFrontend(sinc_filters=256, sinc_kernel_size=251, sinc_stride=16) + >>> input_tensor = torch.randn(32, 16000) # (batch_size, time) + >>> input_lengths = torch.full((32,), 16000) + >>> output, output_lengths = frontend(input_tensor, input_lengths) + >>> print(output.shape) # (batch_size, time', features) + + References: + M. Pariente, S. Cornell, A. Deleforge and E. Vincent, + "Filterbank design for end-to-end speech separation," in Proc. ICASSP, 2020 """ @typechecked @@ -72,14 +92,38 @@ def __init__( def forward( self, input: torch.Tensor, input_length: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply the Asteroid filterbank frontend to the input. + """ + Apply the Asteroid filterbank frontend to the input audio. + + This method processes the input audio through the Asteroid filterbank, + applying preemphasis, normalization, and frame-wise feature extraction. Args: - input: Input (B, T). - input_length: Input length (B,). + input (torch.Tensor): Input audio tensor of shape (B, T), where B is + the batch size and T is the number of time steps. + input_length (torch.Tensor): Tensor of shape (B,) containing the + original lengths of each sequence in the batch. Returns: - Tensor: Frame-wise output (B, T', D). + Tuple[torch.Tensor, torch.Tensor]: + - Frame-wise output features of shape (B, T', D), where T' is the + number of frames and D is the number of features per frame. + - Updated input lengths after processing, of shape (B,). + + Raises: + AssertionError: If the input tensor does not have exactly 2 dimensions. + + Note: + The forward pass temporarily disables automatic mixed precision to + ensure consistent results. + + Example: + >>> frontend = AsteroidFrontend(sinc_filters=256, sinc_kernel_size=251, sinc_stride=16) + >>> input_tensor = torch.randn(32, 16000) # (batch_size, time) + >>> input_lengths = torch.full((32,), 16000) + >>> output, output_lengths = frontend.forward(input_tensor, input_lengths) + >>> print(output.shape) # (batch_size, time', features) + >>> print(output_lengths.shape) # (batch_size,) """ # input check assert ( @@ -106,5 +150,20 @@ def forward( return x.permute(0, 2, 1), input_length def output_size(self) -> int: - """Return output length of feature dimension D.""" + """ + Return the output size of the feature dimension. + + This method returns the number of features in the output of the + Asteroid filterbank frontend, which is equal to the number of + sinc filters used in the convolutional layer. + + Returns: + int: The number of features in the output, corresponding to + the number of sinc filters. + + Example: + >>> frontend = AsteroidFrontend(sinc_filters=256) + >>> output_dim = frontend.output_size() + >>> print(output_dim) # 256 + """ return self.sinc_filters diff --git a/espnet2/asr/frontend/default.py b/espnet2/asr/frontend/default.py index 1cceef269d5..2c76dba7164 100644 --- a/espnet2/asr/frontend/default.py +++ b/espnet2/asr/frontend/default.py @@ -15,9 +15,49 @@ class DefaultFrontend(AbsFrontend): - """Conventional frontend structure for ASR. - - Stft -> WPE -> MVDR-Beamformer -> Power-spec -> Log-Mel-Fbank + """ + Conventional frontend structure for ASR. + + This class implements a standard frontend processing pipeline for Automatic Speech Recognition (ASR), + consisting of the following steps: STFT -> WPE -> MVDR-Beamformer -> Power-spectrum -> Log-Mel-Fbank. + + Attributes: + stft (Stft): Short-time Fourier transform module. + frontend (Frontend): Speech enhancement frontend module. + logmel (LogMel): Log-Mel filterbank feature extraction module. + n_mels (int): Number of Mel filterbank channels. + frontend_type (str): Type of frontend, set to "default". + hop_length (int): Hop length for STFT. + apply_stft (bool): Flag to determine whether to apply STFT. + + Args: + fs (Union[int, str]): Sampling frequency of the input audio. Defaults to 16000. + n_fft (int): FFT size. Defaults to 512. + win_length (Optional[int]): Window length for STFT. Defaults to None. + hop_length (int): Hop length for STFT. Defaults to 128. + window (Optional[str]): Window function type. Defaults to "hann". + center (bool): Whether to pad the input on both sides. Defaults to True. + normalized (bool): Whether to normalize the STFT. Defaults to False. + onesided (bool): Whether to return only one-sided spectrum. Defaults to True. + n_mels (int): Number of Mel filterbank channels. Defaults to 80. + fmin (Optional[int]): Minimum frequency for Mel filters. Defaults to None. + fmax (Optional[int]): Maximum frequency for Mel filters. Defaults to None. + htk (bool): Whether to use HTK formula for Mel scale. Defaults to False. + frontend_conf (Optional[dict]): Configuration for the Frontend module. Defaults to None. + apply_stft (bool): Whether to apply STFT. Defaults to True. + + Note: + This class inherits from AbsFrontend and implements the conventional frontend structure + used in many ASR systems. It combines multiple processing steps to convert raw audio + input into features suitable for acoustic modeling. + + Examples: + >>> frontend = DefaultFrontend(fs=16000, n_fft=512, n_mels=80) + >>> input_audio = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> features, feat_lengths = frontend(input_audio, input_lengths) + >>> features.shape + torch.Size([1, 126, 80]) """ @typechecked @@ -77,11 +117,64 @@ def __init__( self.frontend_type = "default" def output_size(self) -> int: + """ + Returns the output size of the frontend. + + Returns: + int: The number of Mel filterbank channels (n_mels) used in the frontend. + + Note: + This method is used to determine the dimensionality of the feature vectors + produced by the frontend, which is essential for configuring subsequent + components in the ASR pipeline. + + Examples: + >>> frontend = DefaultFrontend(n_mels=80) + >>> frontend.output_size() + 80 + """ return self.n_mels def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform frontend processing on the input audio. + + This method applies the complete frontend processing pipeline to the input audio, + including STFT, optional speech enhancement, channel selection, power spectrum + computation, and log-mel feature extraction. + + Args: + input (torch.Tensor): Input audio tensor of shape (Batch, Time). + input_lengths (torch.Tensor): Tensor of input audio lengths of shape (Batch,). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - input_feats (torch.Tensor): Processed features of shape (Batch, Length, Dim). + - feats_lens (torch.Tensor): Lengths of processed features of shape (Batch,). + + Note: + The processing steps include: + 1. Domain conversion (e.g., STFT) + 2. Optional speech enhancement + 3. Channel selection for multi-channel input + 4. Power spectrum computation + 5. Log-mel feature extraction + + During training, a random channel is selected for multi-channel input. + During inference, the first channel is used. + + Examples: + >>> frontend = DefaultFrontend(fs=16000, n_fft=512, n_mels=80) + >>> input_audio = torch.randn(2, 16000) # 2 utterances of 1 second each + >>> input_lengths = torch.tensor([16000, 12000]) # Second utterance is shorter + >>> features, feat_lengths = frontend(input_audio, input_lengths) + >>> features.shape + torch.Size([2, 126, 80]) + >>> feat_lengths + tensor([126, 95]) + """ # 1. Domain-conversion: e.g. Stft: time -> time-freq if self.stft is not None: input_stft, feats_lens = self._compute_stft(input, input_lengths) diff --git a/espnet2/asr/frontend/fused.py b/espnet2/asr/frontend/fused.py index ab4cd7fdbe8..6bd5762c9b0 100644 --- a/espnet2/asr/frontend/fused.py +++ b/espnet2/asr/frontend/fused.py @@ -10,6 +10,41 @@ class FusedFrontends(AbsFrontend): + """ + A class that combines multiple frontend modules for feature extraction in speech processing. + + This class allows the fusion of different frontend modules, such as DefaultFrontend and S3prlFrontend, + and aligns their outputs using a specified method (currently only linear projection is supported). + + Attributes: + align_method (str): The method used to align and fuse frontend outputs. + proj_dim (int): The dimension of the projection applied to each frontend's output. + frontends (torch.nn.ModuleList): A list of frontend modules to be combined. + gcd (int): The greatest common divisor of hop lengths across all frontends. + factors (list): A list of factors used for reshaping frontend outputs. + projection_layers (torch.nn.ModuleList): Linear projection layers for each frontend. + + Args: + frontends (list): A list of dictionaries, each containing configuration for a frontend. + align_method (str, optional): The method used to align frontend outputs. Defaults to "linear_projection". + proj_dim (int, optional): The dimension of the projection applied to each frontend's output. Defaults to 100. + fs (int, optional): The sampling frequency of the input audio. Defaults to 16000. + + Raises: + NotImplementedError: If an unsupported frontend type is specified. + + Note: + Currently, only 'default' and 's3prl' frontend types are supported. + The 'linear_projection' is the only supported alignment method at the moment. + + Examples: + >>> frontends = [ + ... {"frontend_type": "default", "n_mels": 80}, + ... {"frontend_type": "s3prl", "frontend_conf": "wav2vec2_base"} + ... ] + >>> fused_frontend = FusedFrontends(frontends=frontends, proj_dim=128) + """ + @typechecked def __init__( self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000 @@ -99,11 +134,55 @@ def __init__( self.projection_layers = self.projection_layers.to(torch.device(dev)) def output_size(self) -> int: + """ + Returns the output size of the fused frontends. + + This method calculates the total output size of all combined frontends + after projection and fusion. + + Returns: + int: The total output size, which is the product of the number of + frontends and the projection dimension (proj_dim). + + Example: + >>> fused_frontend = FusedFrontends(frontends=[...], proj_dim=100) + >>> output_size = fused_frontend.output_size() + >>> print(output_size) + 200 # Assuming two frontends are used + """ return len(self.frontends) * self.proj_dim def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Processes the input through all frontends and fuses their outputs. + + This method applies each frontend to the input, projects the outputs, + aligns them, and concatenates them to produce a single fused feature representation. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, num_samples). + input_lengths (torch.Tensor): Tensor of input lengths for each sample in the batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - input_feats (torch.Tensor): The fused features from all frontends. + Shape: (batch_size, num_frames, total_proj_dim). + - feats_lens (torch.Tensor): The lengths of the feature sequences for each sample in the batch. + + Raises: + NotImplementedError: If an unsupported alignment method is specified. + + Note: + Currently, only the 'linear_projection' alignment method is implemented. + + Example: + >>> fused_frontend = FusedFrontends(frontends=[...]) + >>> input_tensor = torch.randn(32, 16000) # (batch_size, num_samples) + >>> input_lengths = torch.full((32,), 16000) + >>> fused_features, feature_lengths = fused_frontend.forward(input_tensor, input_lengths) + """ # step 0 : get all frontends features self.feats = [] for frontend in self.frontends: diff --git a/espnet2/asr/frontend/melspec_torch.py b/espnet2/asr/frontend/melspec_torch.py index 26a1f108f21..0e8d33563cb 100644 --- a/espnet2/asr/frontend/melspec_torch.py +++ b/espnet2/asr/frontend/melspec_torch.py @@ -15,7 +15,32 @@ class MelSpectrogramTorch(AbsFrontend): - """Mel-Spectrogram using Torchaudio Implementation.""" + """ + Mel-Spectrogram using Torchaudio Implementation. + + This class implements a Mel-Spectrogram frontend using Torchaudio's MelSpectrogram + transform. It can optionally apply pre-emphasis, logarithmic scaling, and + normalization to the input audio signal. + + Attributes: + log (bool): Whether to apply logarithmic scaling to the spectrogram. + n_mels (int): Number of Mel filterbanks. + preemp (bool): Whether to apply pre-emphasis to the input signal. + normalize (Optional[str]): Normalization method ('mn' for mean normalization or None). + window_fn (Callable): Window function (torch.hann_window or torch.hamming_window). + flipped_filter (torch.Tensor): Pre-emphasis filter coefficients. + transform (torchaudio.transforms.MelSpectrogram): Mel-spectrogram transform object. + + Note: + This class inherits from AbsFrontend and is designed to be used as a frontend + in speech processing tasks, particularly in the ESPnet2 framework. + + Example: + >>> frontend = MelSpectrogramTorch(n_mels=80, n_fft=512, win_length=400, hop_length=160) + >>> input_signal = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> mel_spec, output_lengths = frontend(input_signal, input_lengths) + """ @typechecked def __init__( @@ -63,6 +88,37 @@ def __init__( def forward( self, input: torch.Tensor, input_length: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Compute the Mel-spectrogram of the input audio signal. + + This method applies the Mel-spectrogram transformation to the input audio signal, + optionally performing pre-emphasis, logarithmic scaling, and normalization. + + Args: + input (torch.Tensor): Input audio signal tensor of shape (batch_size, num_samples). + input_length (torch.Tensor): Tensor containing the lengths of each audio signal + in the batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - torch.Tensor: Mel-spectrogram features of shape (batch_size, num_frames, n_mels). + - torch.Tensor: Output lengths tensor containing the number of frames for each + spectrogram in the batch. + + Raises: + AssertionError: If the input tensor does not have exactly 2 dimensions. + + Note: + This method uses torch.no_grad() and torch.cuda.amp.autocast(enabled=False) + to ensure consistent behavior and avoid unnecessary gradient computations. + + Example: + >>> frontend = MelSpectrogramTorch(n_mels=80, n_fft=512, win_length=400, hop_length=160) + >>> input_signal = torch.randn(2, 16000) # Batch of 2 audio signals + >>> input_lengths = torch.tensor([16000, 16000]) + >>> mel_spec, output_lengths = frontend(input_signal, input_lengths) + >>> print(mel_spec.shape) # Expected output: torch.Size([2, num_frames, 80]) + """ # input check assert ( len(input.size()) == 2 @@ -98,5 +154,20 @@ def forward( return x.permute(0, 2, 1), input_length def output_size(self) -> int: - """Return output length of feature dimension D.""" + """ + Return the output size of the Mel-spectrogram feature dimension. + + This method returns the number of Mel filterbanks used in the Mel-spectrogram + computation, which corresponds to the size of the feature dimension in the + output. + + Returns: + int: The number of Mel filterbanks (n_mels) used in the Mel-spectrogram + computation. + + Example: + >>> frontend = MelSpectrogramTorch(n_mels=80) + >>> feature_dim = frontend.output_size() + >>> print(feature_dim) # Expected output: 80 + """ return self.n_mels diff --git a/espnet2/asr/frontend/s3prl.py b/espnet2/asr/frontend/s3prl.py index 39ccefb56a4..191957f5ab2 100644 --- a/espnet2/asr/frontend/s3prl.py +++ b/espnet2/asr/frontend/s3prl.py @@ -12,7 +12,40 @@ class S3prlFrontend(AbsFrontend): - """Speech Pretrained Representation frontend structure for ASR.""" + """ + Speech Pretrained Representation frontend structure for ASR. + + This class implements a frontend for Automatic Speech Recognition (ASR) using + pretrained speech representations from the S3PRL toolkit. It supports various + upstream models and allows for flexible configuration of the frontend. + + Attributes: + frontend_type (str): Type of the frontend, set to "s3prl". + hop_length (int): The hop length (downsampling rate) of the featurizer. + tile_factor (int): Factor by which to tile the output representations. + multilayer_feature (bool): Whether to use multilayer features or not. + layer (int): Specific layer to use from the upstream model (-1 for last layer). + + Args: + fs (Union[int, str]): Sampling frequency of the input audio. Defaults to 16000. + frontend_conf (Optional[dict]): Configuration for the frontend. Defaults to None. + download_dir (Optional[str]): Directory to download S3PRL models. Defaults to None. + multilayer_feature (bool): Whether to use multilayer features. Defaults to False. + layer (int): Specific layer to use from the upstream model. Defaults to -1. + + Raises: + Exception: If S3PRL is not properly installed. + + Note: + - All upstream models in S3PRL currently only support 16 kHz audio. + - When a specific layer is selected, multilayer_feature will be deactivated. + + Examples: + >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) + >>> input_tensor = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> feats, feats_lens = frontend(input_tensor, input_lengths) + """ @typechecked def __init__( @@ -91,11 +124,53 @@ def _tile_representations(self, feature): return tiled_feature def output_size(self) -> int: + """ + Returns the output size of the frontend. + + This method provides the dimensionality of the feature vectors produced by the frontend. + + Returns: + int: The size of the output feature vectors. + + Example: + >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) + >>> output_dim = frontend.output_size() + >>> print(output_dim) + 768 # Example output, actual value may vary depending on the upstream model + """ return self.featurizer.output_size def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Processes the input audio and returns the extracted features. + + This method takes the input audio tensor and its corresponding lengths, passes it through + the S3PRL upstream model and featurizer, and returns the extracted features. + + Args: + input (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). + input_lengths (torch.Tensor): Lengths of each audio in the batch of shape (batch_size,). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - feats (torch.Tensor): Extracted features of shape (batch_size, num_frames, feature_dim). + - feats_lens (torch.Tensor): Lengths of each feature sequence in the batch of shape (batch_size,). + + Note: + - If a specific layer is selected (self.layer != -1), only features from that layer are returned. + - If multilayer_feature is True, features from multiple layers are combined. + - If tile_factor is not 1, the output features are tiled accordingly. + + Example: + >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) + >>> input_tensor = torch.randn(2, 16000) # 2 audio samples of 1 second each + >>> input_lengths = torch.tensor([16000, 16000]) + >>> feats, feats_lens = frontend(input_tensor, input_lengths) + >>> print(feats.shape) + torch.Size([2, 50, 768]) # Example output, actual shape may vary + """ feats, feats_lens = self.upstream(input, input_lengths) if self.layer != -1: layer = self.layer @@ -113,5 +188,20 @@ def forward( return feats, feats_lens def reload_pretrained_parameters(self): - self.upstream.load_state_dict(self.pretrained_params) + """ + Reloads the pretrained parameters of the S3PRL frontend model. + + This method restores the original pretrained parameters of the upstream S3PRL model, + effectively resetting any fine-tuning or modifications made to the model weights. + + Note: + This method is useful when you want to reset the model to its initial pretrained state, + for example, after fine-tuning or experimenting with the model parameters. + + Example: + >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) + >>> # After some operations or fine-tuning + >>> frontend.reload_pretrained_parameters() + >>> print("Pretrained S3PRL frontend model parameters reloaded!") + """ logging.info("Pretrained S3PRL frontend model parameters reloaded!") diff --git a/espnet2/asr/frontend/whisper.py b/espnet2/asr/frontend/whisper.py index b879d606a42..35ec4b4fdea 100644 --- a/espnet2/asr/frontend/whisper.py +++ b/espnet2/asr/frontend/whisper.py @@ -10,9 +10,41 @@ class WhisperFrontend(AbsFrontend): - """Speech Representation Using Encoder Outputs from OpenAI's Whisper Model: + """ + Speech representation frontend using OpenAI's Whisper model encoder outputs. + + This class implements a frontend for speech representation using the encoder + outputs from OpenAI's Whisper model. It processes input audio to generate + log-mel spectrograms and then encodes them using the Whisper encoder. + + Attributes: + n_fft (int): Size of the FFT for spectrogram computation. + win_length (int): Window length for spectrogram computation. + hop_length (int): Hop length for spectrogram computation. + n_mels (int): Number of mel filterbanks. + mel_filters (function): Function to generate mel filterbanks. + pad_or_trim (function): Function to pad or trim input sequences. + whisper (whisper.Whisper): Loaded Whisper model. + freeze_weights (bool): Whether to freeze Whisper model weights. + + Args: + whisper_model (str): Name of the Whisper model to use. Defaults to "small". + fs (Union[int, str]): Sampling frequency in Hz. Defaults to 16000. + freeze_weights (bool): Whether to freeze Whisper model weights. Defaults to True. + download_dir (Optional[str]): Directory to download Whisper model. Defaults to None. + + Raises: + Exception: If the whisper package is not properly installed. + + Note: + This class requires the whisper package to be installed. + The Whisper model only supports 16 kHz audio. - URL: https://github.com/openai/whisper + Examples: + >>> frontend = WhisperFrontend("small", fs=16000) + >>> input_tensor = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> output, output_lengths = frontend(input_tensor, input_lengths) """ @typechecked @@ -58,6 +90,21 @@ def __init__( self.freeze_weights = freeze_weights def output_size(self) -> int: + """ + Returns the output size of the Whisper frontend. + + This method returns the dimensionality of the feature vectors produced by + the Whisper encoder. + + Returns: + int: The size of the output feature vectors. + + Example: + >>> frontend = WhisperFrontend("small") + >>> output_dim = frontend.output_size() + >>> print(output_dim) + 512 # This may vary depending on the Whisper model used + """ return self.whisper.encoder.ln_post.normalized_shape[-1] def log_mel_spectrogram( @@ -65,6 +112,34 @@ def log_mel_spectrogram( audio: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: + """ + Compute log-mel spectrogram features from input audio. + + This method computes the log-mel spectrogram features from the input audio + using the Whisper model's preprocessing steps. + + Args: + audio (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). + ilens (torch.Tensor, optional): Tensor of input lengths for each audio in the batch. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - log_spec (torch.Tensor): Log-mel spectrogram features of shape + (batch_size, n_mels, num_frames). + - olens (Optional[torch.Tensor]): Tensor of output lengths for each + spectrogram in the batch. None if ilens is None. + + Note: + The method applies normalization to the log-mel spectrogram as per + Whisper's preprocessing. + + Example: + >>> frontend = WhisperFrontend("small") + >>> audio = torch.randn(1, 16000) + >>> log_spec, olens = frontend.log_mel_spectrogram(audio) + >>> print(log_spec.shape) + torch.Size([1, 80, 100]) # Exact shape may vary based on input length + """ window = torch.hann_window(self.win_length).to(audio.device) stft = torch.stft( audio, self.n_fft, self.hop_length, window=window, return_complex=True @@ -96,6 +171,34 @@ def whisper_encode( input: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: + """ + Encode input features using the Whisper encoder. + + This method processes the input features (typically log-mel spectrograms) + through the Whisper encoder to produce high-level representations. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, n_mels, num_frames). + ilens (torch.Tensor, optional): Tensor of input lengths for each feature in the batch. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - x (torch.Tensor): Encoded features of shape (batch_size, num_frames, encoder_dim). + - olens (Optional[torch.Tensor]): Tensor of output lengths for each encoded + sequence in the batch. None if ilens is None. + + Note: + The method applies positional embeddings and processes the input through + the Whisper encoder blocks. The output is truncated if it exceeds the + maximum position embedding size. + + Example: + >>> frontend = WhisperFrontend("small") + >>> log_spec = torch.randn(1, 80, 100) + >>> encoded, olens = frontend.whisper_encode(log_spec) + >>> print(encoded.shape) + torch.Size([1, 100, 512]) # Exact shape may vary based on the Whisper model + """ whisper_encoder = self.whisper.encoder x = F.gelu(whisper_encoder.conv1(input)) @@ -133,6 +236,33 @@ def whisper_encode( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Process input audio through the Whisper frontend. + + This method takes raw audio input, computes log-mel spectrograms, and then + encodes them using the Whisper encoder to produce high-level speech representations. + + Args: + input (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). + input_lengths (torch.Tensor): Tensor of input lengths for each audio in the batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - feats (torch.Tensor): Encoded features of shape (batch_size, num_frames, encoder_dim). + - feats_lens (torch.Tensor): Tensor of output lengths for each encoded sequence in the batch. + + Note: + If self.freeze_weights is True, the Whisper encoding step is performed + with torch.no_grad() to prevent gradient computation and weight updates. + + Example: + >>> frontend = WhisperFrontend("small") + >>> audio = torch.randn(1, 16000) + >>> input_lengths = torch.tensor([16000]) + >>> feats, feats_lens = frontend(audio, input_lengths) + >>> print(feats.shape) + torch.Size([1, 100, 512]) # Exact shape may vary based on input and Whisper model + """ feats, feats_lens = self.log_mel_spectrogram(input, input_lengths) with torch.no_grad() if self.freeze_weights else contextlib.nullcontext(): diff --git a/espnet2/asr/frontend/windowing.py b/espnet2/asr/frontend/windowing.py index f4a34d68e94..751148b8bf5 100644 --- a/espnet2/asr/frontend/windowing.py +++ b/espnet2/asr/frontend/windowing.py @@ -13,17 +13,35 @@ class SlidingWindow(AbsFrontend): - """Sliding Window. + """ + Sliding Window for raw audio input data. + + This class implements a sliding window operation over batched continuous raw audio tensors. + It is designed to be used as a frontend in audio processing pipelines, particularly + in combination with pre-encoders compatible with raw audio data (e.g., Sinc convolutions). + + The sliding window operation segments the input audio into overlapping frames, + which can be further processed by subsequent layers in the neural network. + + Attributes: + win_length (int): Length of each window frame. + hop_length (int): Number of samples to advance between successive frames. + channels (int): Number of input audio channels. + padding (Optional[int]): Padding option (currently not implemented). + fs (Optional[int]): Sampling rate (placeholder for compatibility, not used). - Provides a sliding window over a batched continuous raw audio tensor. - Optionally, provides padding (Currently not implemented). - Combine this module with a pre-encoder compatible with raw audio data, - for example Sinc convolutions. + Note: + - Output length is calculated incorrectly if audio is shorter than win_length. + - Trailing values are currently discarded as padding is not yet implemented. + - No additional window function is applied to input values. - Known issues: - Output length is calculated incorrectly if audio shorter than win_length. - WARNING: trailing values are discarded - padding not implemented yet. - There is currently no additional window function applied to input values. + Examples: + >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) + >>> input_tensor = torch.randn(32, 16000, 1) # (batch_size, time_steps, channels) + >>> input_lengths = torch.full((32,), 16000) + >>> output, output_lengths = sliding_window(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 98, 1, 400]) """ @typechecked @@ -54,15 +72,38 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply a sliding window on the input. + """ + Apply a sliding window on the input. + + This method processes the input audio tensor by applying a sliding window, + segmenting the continuous audio into overlapping frames. Args: - input: Input (B, T, C*D) or (B, T*C*D), with D=C=1. - input_lengths: Input lengths within batch. + input (torch.Tensor): Input tensor of shape (B, T, C*D) or (B, T*C*D), + where B is batch size, T is time steps, C is channels, and D is 1. + input_lengths (torch.Tensor): Tensor containing the valid length of each + sample in the batch. Returns: - Tensor: Output with dimensions (B, T, C, D), with D=win_length. - Tensor: Output lengths within batch. + Tuple[torch.Tensor, torch.Tensor]: + - output (torch.Tensor): Windowed output of shape (B, T, C, D), + where D is the window length (win_length). + - output_lengths (torch.Tensor): Tensor containing the valid length + of each windowed sample in the batch. + + Examples: + >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) + >>> input_tensor = torch.randn(32, 16000, 1) # (batch_size, time_steps, channels) + >>> input_lengths = torch.full((32,), 16000) + >>> output, output_lengths = sliding_window.forward(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 98, 1, 400]) + >>> print(output_lengths) + tensor([98, 98, 98, ..., 98, 98, 98]) + + Note: + The output length is calculated as: + (input_length - win_length) // hop_length + 1 """ input_size = input.size() B = input_size[0] @@ -84,5 +125,22 @@ def forward( return output, output_lengths def output_size(self) -> int: - """Return output length of feature dimension D, i.e. the window length.""" + """ + Return the output size of the feature dimension. + + This method returns the length of the window, which corresponds to the + size of the feature dimension (D) in the output tensor. + + Returns: + int: The window length (win_length). + + Examples: + >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) + >>> print(sliding_window.output_size()) + 400 + + Note: + This value is used to determine the size of the last dimension in the + output tensor returned by the forward method. + """ return self.win_length diff --git a/espnet2/asr/layers/cgmlp.py b/espnet2/asr/layers/cgmlp.py index c404bc3828a..3551699c12d 100644 --- a/espnet2/asr/layers/cgmlp.py +++ b/espnet2/asr/layers/cgmlp.py @@ -13,7 +13,39 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module): - """Convolutional Spatial Gating Unit (CSGU).""" + """ + Convolutional Spatial Gating Unit (CSGU) for MLP with convolutional gating. + + This class implements the Convolutional Spatial Gating Unit, a key component + of the cgMLP architecture. It applies layer normalization, 1D convolution, + and optional linear transformation to the input, followed by an activation + function and dropout. + + Attributes: + norm (LayerNorm): Layer normalization applied to the input. + conv (torch.nn.Conv1d): 1D convolution layer. + linear (torch.nn.Linear or None): Optional linear layer after convolution. + act (torch.nn.Module): Activation function for gating. + dropout (torch.nn.Dropout): Dropout layer. + + Args: + size (int): Input size (should be divisible by 2). + kernel_size (int): Kernel size for the 1D convolution. + dropout_rate (float): Dropout rate. + use_linear_after_conv (bool): Whether to use a linear layer after convolution. + gate_activation (str): Activation function to use for gating. + + Note: + The input channels are split in half, with one half used for gating + the other half. + + Example: + >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') + >>> input_tensor = torch.randn(32, 100, 512) + >>> output = csgu(input_tensor) + >>> output.shape + torch.Size([32, 100, 256]) + """ def __init__( self, @@ -48,21 +80,66 @@ def __init__( self.dropout = torch.nn.Dropout(dropout_rate) def espnet_initialization_fn(self): - torch.nn.init.normal_(self.conv.weight, std=1e-6) + """ + Initialize the weights of the convolutional and linear layers. + + This method initializes the weights of the convolutional layer and, if present, + the linear layer of the Convolutional Spatial Gating Unit (CSGU). The + initialization follows the strategy described in the original cgMLP paper. + + The convolutional layer's weights are initialized from a normal distribution + with a small standard deviation, while its biases are initialized to ones. + If a linear layer is present, its weights are similarly initialized from a + normal distribution, and its biases are set to ones. + + Returns: + None + + Note: + This method is typically called during the initialization of the CSGU + module and doesn't need to be called manually in most cases. + + Example: + >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') + >>> csgu.espnet_initialization_fn() + """ torch.nn.init.ones_(self.conv.bias) if self.linear is not None: torch.nn.init.normal_(self.linear.weight, std=1e-6) torch.nn.init.ones_(self.linear.bias) def forward(self, x, gate_add=None): - """Forward method + """ + Forward pass of the Convolutional Spatial Gating Unit. + + This method performs the forward computation of the CSGU. It splits the input + tensor into two halves along the last dimension, applies normalization and + convolution to one half, and uses it to gate the other half. Args: - x (torch.Tensor): (N, T, D) - gate_add (torch.Tensor): (N, T, D/2) + x (torch.Tensor): Input tensor of shape (N, T, D), where N is the batch size, + T is the sequence length, and D is the feature dimension. + gate_add (torch.Tensor, optional): Additional tensor to be added to the gate + values. Should have shape (N, T, D/2). Defaults to None. Returns: - out (torch.Tensor): (N, T, D/2) + torch.Tensor: Output tensor of shape (N, T, D/2). The output dimension is + half of the input dimension due to the gating mechanism. + + Raises: + ValueError: If the input tensor's last dimension is not even. + + Example: + >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') + >>> input_tensor = torch.randn(32, 100, 512) + >>> output = csgu(input_tensor) + >>> output.shape + torch.Size([32, 100, 256]) + + Note: + The input tensor is split into two halves: x_r and x_g. x_g is processed + through normalization, convolution, and optional linear transformation, + then used to gate x_r. The result is then passed through dropout. """ x_r, x_g = x.chunk(2, dim=-1) @@ -82,7 +159,42 @@ def forward(self, x, gate_add=None): class ConvolutionalGatingMLP(torch.nn.Module): - """Convolutional Gating MLP (cgMLP).""" + """ + Convolutional Gating MLP (cgMLP) module. + + This class implements the Convolutional Gating MLP, a variant of the MLP + architecture that incorporates convolutional gating mechanisms. It consists + of two channel projection layers and a Convolutional Spatial Gating Unit (CSGU). + + The cgMLP processes input by first projecting it to a higher dimension, + then applying the CSGU for feature interaction and gating, and finally + projecting it back to the original dimension. + + Attributes: + channel_proj1 (torch.nn.Sequential): First channel projection layer with GELU activation. + csgu (ConvolutionalSpatialGatingUnit): The core CSGU component. + channel_proj2 (torch.nn.Linear): Second channel projection layer. + + Args: + size (int): Input and output size of the module. + linear_units (int): Number of units in the intermediate linear layer. + kernel_size (int): Kernel size for the convolutional layer in CSGU. + dropout_rate (float): Dropout rate used in CSGU. + use_linear_after_conv (bool): Whether to use a linear layer after convolution in CSGU. + gate_activation (str): Activation function to use for gating in CSGU. + + Example: + >>> cgmlp = ConvolutionalGatingMLP(512, 1024, 3, 0.1, True, 'gelu') + >>> input_tensor = torch.randn(32, 100, 512) + >>> output = cgmlp(input_tensor, None) + >>> output.shape + torch.Size([32, 100, 512]) + + Note: + The forward method of this class can handle inputs with or without + positional embeddings. If positional embeddings are provided, they + will be returned along with the processed tensor. + """ def __init__( self, @@ -108,6 +220,44 @@ def __init__( self.channel_proj2 = torch.nn.Linear(linear_units // 2, size) def forward(self, x, mask): + """ + Forward pass of the Convolutional Gating MLP. + + This method performs the forward computation of the cgMLP. It processes + the input through two channel projection layers and a Convolutional + Spatial Gating Unit (CSGU). + + Args: + x (torch.Tensor or tuple): If tuple, it should contain: + - xs_pad (torch.Tensor): Input tensor of shape (N, T, D), where N is + the batch size, T is the sequence length, and D is the feature dimension. + - pos_emb (torch.Tensor): Positional embedding tensor. + If not a tuple, it's treated as the input tensor itself. + mask (torch.Tensor): Mask tensor (not used in the current implementation). + + Returns: + torch.Tensor or tuple: If the input included positional embeddings, returns a tuple + containing: + - xs_pad (torch.Tensor): Processed tensor of shape (N, T, D). + - pos_emb (torch.Tensor): The input positional embedding tensor. + Otherwise, returns just the processed tensor. + + Example: + >>> cgmlp = ConvolutionalGatingMLP(512, 1024, 3, 0.1, True, 'gelu') + >>> input_tensor = torch.randn(32, 100, 512) + >>> output = cgmlp(input_tensor, None) + >>> output.shape + torch.Size([32, 100, 512]) + + >>> input_with_pos = (input_tensor, torch.randn(32, 100, 512)) + >>> output_with_pos = cgmlp(input_with_pos, None) + >>> isinstance(output_with_pos, tuple) + True + + Note: + The 'mask' argument is currently not used in the method but is kept + for compatibility with other modules in the architecture. + """ if isinstance(x, tuple): xs_pad, pos_emb = x else: diff --git a/espnet2/asr/layers/fastformer.py b/espnet2/asr/layers/fastformer.py index 24ca9475c15..6cfefb476a9 100644 --- a/espnet2/asr/layers/fastformer.py +++ b/espnet2/asr/layers/fastformer.py @@ -12,7 +12,44 @@ class FastSelfAttention(torch.nn.Module): - """Fast self-attention used in Fastformer.""" + """ + Fast self-attention mechanism used in Fastformer. + + This class implements the fast self-attention mechanism as described in the + Fastformer paper by Wu et al. It provides an efficient alternative to + traditional self-attention mechanisms used in transformer models. + + Attributes: + attention_head_size (int): The size of each attention head. + num_attention_heads (int): The number of attention heads. + query (torch.nn.Linear): Linear layer for query transformation. + query_att (torch.nn.Linear): Linear layer for query attention. + key (torch.nn.Linear): Linear layer for key transformation. + key_att (torch.nn.Linear): Linear layer for key attention. + transform (torch.nn.Linear): Linear layer for final transformation. + dropout (torch.nn.Dropout): Dropout layer for regularization. + + Args: + size (int): The input size (hidden size) of the model. + attention_heads (int): The number of attention heads. + dropout_rate (float): The dropout rate to use for regularization. + + Raises: + ValueError: If the hidden size is not an integer multiple of the number + of attention heads. + + Example: + >>> model = FastSelfAttention(512, 8, 0.1) + >>> input_tensor = torch.randn(32, 100, 512) # (batch, seq_len, hidden_size) + >>> mask = torch.ones(32, 1, 100) # (batch, 1, seq_len) + >>> output = model(input_tensor, mask) + >>> print(output.shape) + torch.Size([32, 100, 512]) + + Note: + This implementation is based on the paper "Fastformer: Additive Attention + Can Be All You Need" by Wu et al. and the corresponding GitHub repository. + """ def __init__( self, @@ -37,22 +74,72 @@ def __init__( self.dropout = torch.nn.Dropout(dropout_rate) def espnet_initialization_fn(self): - self.apply(self.init_weights) + """ + Initialize the weights of the FastSelfAttention module. + + This method applies the `init_weights` function to all submodules of the + FastSelfAttention module. It is designed to be used as an initialization + function in the ESPnet framework. + + Note: + This method does not take any arguments and does not return anything. + It modifies the module's weights in-place. + + Example: + >>> model = FastSelfAttention(512, 8, 0.1) + >>> model.espnet_initialization_fn() + """ def init_weights(self, module): + """ + Initialize the weights of a given module. + + This method initializes the weights of linear layers in the module. It sets + the weights to a normal distribution with mean 0.0 and standard deviation 0.02, + and initializes biases to zero. + + Args: + module (torch.nn.Module): The module whose weights are to be initialized. + + Note: + This method is typically called by `espnet_initialization_fn` and is not + meant to be used directly in most cases. + + Example: + >>> linear = torch.nn.Linear(10, 10) + >>> model = FastSelfAttention(512, 8, 0.1) + >>> model.init_weights(linear) + """ if isinstance(module, torch.nn.Linear): module.weight.data.normal_(mean=0.0, std=0.02) if isinstance(module, torch.nn.Linear) and module.bias is not None: module.bias.data.zero_() def transpose_for_scores(self, x): - """Reshape and transpose to compute scores. + """ + Reshape and transpose the input tensor for computing attention scores. + + This method reshapes the input tensor and transposes it to prepare for + computing attention scores in the fast self-attention mechanism. Args: - x: (batch, time, size = n_heads * attn_dim) + x (torch.Tensor): Input tensor of shape (batch, time, size), + where size = n_heads * attn_dim. Returns: - (batch, n_heads, time, attn_dim) + torch.Tensor: Reshaped and transposed tensor of shape + (batch, n_heads, time, attn_dim). + + Example: + >>> model = FastSelfAttention(512, 8, 0.1) + >>> x = torch.randn(32, 100, 512) + >>> transposed = model.transpose_for_scores(x) + >>> print(transposed.shape) + torch.Size([32, 8, 100, 64]) + + Note: + This method is used internally in the forward pass of the FastSelfAttention + module and is not typically called directly by users. """ new_x_shape = x.shape[:-1] + ( @@ -62,14 +149,34 @@ def transpose_for_scores(self, x): return x.reshape(*new_x_shape).transpose(1, 2) def forward(self, xs_pad, mask): - """Forward method. + """ + Perform the forward pass of the FastSelfAttention module. + + This method implements the fast self-attention mechanism, processing the input + tensor and applying the attention weights to produce the output. Args: - xs_pad: (batch, time, size = n_heads * attn_dim) - mask: (batch, 1, time), nonpadding is 1, padding is 0 + xs_pad (torch.Tensor): Input tensor of shape (batch, time, size), + where size = n_heads * attn_dim. + mask (torch.Tensor): Mask tensor of shape (batch, 1, time), where + non-padding positions are 1 and padding positions are 0. Returns: - torch.Tensor: (batch, time, size) + torch.Tensor: Output tensor after applying fast self-attention, + of shape (batch, time, size). + + Example: + >>> model = FastSelfAttention(512, 8, 0.1) + >>> xs_pad = torch.randn(32, 100, 512) + >>> mask = torch.ones(32, 1, 100) + >>> output = model.forward(xs_pad, mask) + >>> print(output.shape) + torch.Size([32, 100, 512]) + + Note: + The mask tensor should have 1s for non-padding positions and 0s for + padding positions. The method internally inverts this mask for + compatibility with the implementation. """ batch_size, seq_len, _ = xs_pad.shape diff --git a/espnet2/asr/maskctc_model.py b/espnet2/asr/maskctc_model.py index b3960359340..499767c30d5 100644 --- a/espnet2/asr/maskctc_model.py +++ b/espnet2/asr/maskctc_model.py @@ -37,7 +37,53 @@ def autocast(enabled=True): class MaskCTCModel(ESPnetASRModel): - """Hybrid CTC/Masked LM Encoder-Decoder model (Mask-CTC)""" + """ + Hybrid CTC/Masked LM Encoder-Decoder model (Mask-CTC). + + This class implements the Mask-CTC model, which combines Connectionist Temporal + Classification (CTC) and Masked Language Model (MLM) approaches for automatic + speech recognition (ASR). + + The model consists of an encoder, a CTC branch, and an MLM decoder branch. It + supports various components such as frontend processing, spectrogram + augmentation, normalization, pre-encoding, post-encoding, and joint network + functionalities. + + Attributes: + vocab_size (int): Size of the vocabulary, including the mask token. + token_list (List[str]): List of tokens in the vocabulary. + mask_token (int): Token ID for the mask token. + criterion_mlm (LabelSmoothingLoss): Loss function for the MLM branch. + error_calculator (ErrorCalculator): Calculator for CER and WER metrics. + + Args: + vocab_size (int): Size of the vocabulary. + token_list (Union[Tuple[str, ...], List[str]]): List of tokens in the vocabulary. + frontend (Optional[AbsFrontend]): Frontend processing module. + specaug (Optional[AbsSpecAug]): Spectrogram augmentation module. + normalize (Optional[AbsNormalize]): Normalization module. + preencoder (Optional[AbsPreEncoder]): Pre-encoder module. + encoder (AbsEncoder): Encoder module. + postencoder (Optional[AbsPostEncoder]): Post-encoder module. + decoder (MLMDecoder): Masked Language Model decoder. + ctc (CTC): Connectionist Temporal Classification module. + joint_network (Optional[torch.nn.Module]): Joint network module. + ctc_weight (float): Weight for the CTC loss (default: 0.5). + interctc_weight (float): Weight for intermediate CTC loss (default: 0.0). + ignore_id (int): ID to be ignored in loss computation (default: -1). + lsm_weight (float): Label smoothing weight (default: 0.0). + length_normalized_loss (bool): Whether to normalize loss by length (default: False). + report_cer (bool): Whether to report Character Error Rate (default: True). + report_wer (bool): Whether to report Word Error Rate (default: True). + sym_space (str): Space symbol (default: ""). + sym_blank (str): Blank symbol (default: ""). + sym_mask (str): Mask symbol (default: ""). + extract_feats_in_collect_stats (bool): Whether to extract features in collect_stats (default: True). + + Note: + This model extends the ESPnetASRModel and modifies it to incorporate + the Mask-CTC approach, which combines CTC and MLM for improved ASR performance. + """ @typechecked def __init__( @@ -120,13 +166,41 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss + """ + Forward pass for the Mask-CTC model. + + This method performs the forward pass through the entire model, including + frontend processing, encoding, CTC loss calculation, and MLM loss calculation. Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) + speech (torch.Tensor): Input speech tensor of shape (Batch, Length, ...). + speech_lengths (torch.Tensor): Tensor of input speech lengths of shape (Batch,). + text (torch.Tensor): Target text tensor of shape (Batch, Length). + text_lengths (torch.Tensor): Tensor of target text lengths of shape (Batch,). + **kwargs: Additional keyword arguments. + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: A tuple containing: + - loss (torch.Tensor): The total loss for the forward pass. + - stats (Dict[str, torch.Tensor]): A dictionary of statistics including + CTC loss, MLM loss, accuracies, and error rates. + - weight (torch.Tensor): The batch size as a weight for the loss. + + Raises: + AssertionError: If the input tensors have inconsistent batch sizes or + if text_lengths is not a 1D tensor. + + Note: + This method calculates both CTC and MLM losses, combining them based on + the specified CTC weight. It also handles intermediate CTC loss if enabled. + + Examples: + >>> speech = torch.randn(2, 1000, 80) + >>> speech_lengths = torch.tensor([1000, 800]) + >>> text = torch.randint(0, 100, (2, 20)) + >>> text_lengths = torch.tensor([20, 15]) + >>> model = MaskCTCModel(...) + >>> loss, stats, weight = model.forward(speech, speech_lengths, text, text_lengths) """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -246,6 +320,28 @@ def nll( ys_pad: torch.Tensor, ys_pad_lens: torch.Tensor, ) -> torch.Tensor: + """ + Calculate negative log-likelihood for the Mask-CTC model. + + This method is not implemented for the Mask-CTC model and will raise a + NotImplementedError if called. + + Args: + encoder_out (torch.Tensor): Output from the encoder. + encoder_out_lens (torch.Tensor): Lengths of encoder outputs. + ys_pad (torch.Tensor): Padded target sequences. + ys_pad_lens (torch.Tensor): Lengths of target sequences. + + Returns: + torch.Tensor: This method does not return a value. + + Raises: + NotImplementedError: This method is not implemented for the Mask-CTC model. + + Note: + The negative log-likelihood calculation is not applicable to the Mask-CTC + model due to its hybrid nature combining CTC and MLM approaches. + """ raise NotImplementedError def batchify_nll( @@ -256,11 +352,63 @@ def batchify_nll( ys_pad_lens: torch.Tensor, batch_size: int = 100, ): + """ + Batchify the calculation of negative log-likelihood for the Mask-CTC model. + + This method is not implemented for the Mask-CTC model and will raise a + NotImplementedError if called. + + Args: + encoder_out (torch.Tensor): Output from the encoder. + encoder_out_lens (torch.Tensor): Lengths of encoder outputs. + ys_pad (torch.Tensor): Padded target sequences. + ys_pad_lens (torch.Tensor): Lengths of target sequences. + batch_size (int): Size of each batch for processing (default: 100). + + Returns: + This method does not return a value. + + Raises: + NotImplementedError: This method is not implemented for the Mask-CTC model. + + Note: + The batchified negative log-likelihood calculation is not applicable to the + Mask-CTC model due to its hybrid nature combining CTC and MLM approaches. + This method is included for compatibility with other ASR model interfaces + but is not functional in the Mask-CTC context. + """ raise NotImplementedError class MaskCTCInference(torch.nn.Module): - """Mask-CTC-based non-autoregressive inference""" + """ + Mask-CTC-based non-autoregressive inference for automatic speech recognition. + + This class implements the inference process for the Mask-CTC model, which combines + Connectionist Temporal Classification (CTC) and Masked Language Model (MLM) approaches + for non-autoregressive speech recognition. + + The inference process involves iterative decoding, where masked tokens are + progressively predicted based on CTC probabilities and MLM predictions. + + Attributes: + ctc (CTC): The CTC module from the ASR model. + mlm (MLMDecoder): The MLM decoder from the ASR model. + mask_token (int): The token ID representing the mask. + n_iterations (int): Number of iterations for the decoding process. + threshold_probability (float): Probability threshold for masking tokens. + converter (TokenIDConverter): Converter for token IDs to text. + + Args: + asr_model (MaskCTCModel): The trained Mask-CTC ASR model. + n_iterations (int): Number of iterations for the decoding process. + threshold_probability (float): Probability threshold for masking tokens. + + Note: + This class is designed to work with the MaskCTCModel and provides a + non-autoregressive inference method that can potentially be faster than + traditional autoregressive decoding approaches. + """ def __init__( self, @@ -278,11 +426,62 @@ def __init__( self.converter = TokenIDConverter(token_list=asr_model.token_list) def ids2text(self, ids: List[int]): + """ + Convert a list of token IDs to readable text. + + This method converts a sequence of token IDs to a human-readable string, + replacing special tokens with their corresponding symbols. + + Args: + ids (List[int]): A list of token IDs to be converted to text. + + Returns: + str: The converted text string. + + Note: + - The method replaces "" with "_" and "" with a space character. + - This conversion is useful for visualizing the output during the inference process. + + Example: + >>> inference = MaskCTCInference(...) + >>> ids = [1, 2, 3, 4, 5] # Assuming these are valid token IDs + >>> text = inference.ids2text(ids) + >>> print(text) + 'converted text' + """ text = "".join(self.converter.ids2tokens(ids)) return text.replace("", "_").replace("", " ") def forward(self, enc_out: torch.Tensor) -> List[Hypothesis]: - """Perform Mask-CTC inference""" + """ + Perform Mask-CTC inference on the given encoder output. + + This method implements the non-autoregressive Mask-CTC inference algorithm, + which iteratively refines the output by predicting masked tokens. + + Args: + enc_out (torch.Tensor): The encoder output tensor of shape (T, D), + where T is the sequence length and D is the feature dimension. + + Returns: + List[Hypothesis]: A list containing a single Hypothesis object with the + predicted token sequence. + + Note: + The inference process involves the following steps: + 1. Generate initial CTC greedy outputs + 2. Mask low-confidence tokens based on CTC probabilities + 3. Iteratively predict masked tokens using the MLM decoder + 4. Finalize the output by predicting any remaining masked tokens + + The method logs intermediate results for debugging purposes. + + Example: + >>> inference = MaskCTCInference(...) + >>> encoder_output = torch.randn(100, 256) # (T, D) + >>> hypothesis = inference(encoder_output) + >>> predicted_tokens = hypothesis[0].yseq + """ # greedy ctc outputs enc_out = enc_out.unsqueeze(0) ctc_probs, ctc_ids = torch.exp(self.ctc.log_softmax(enc_out)).max(dim=-1) diff --git a/espnet2/asr/pit_espnet_model.py b/espnet2/asr/pit_espnet_model.py index aa62abbc471..8e1734e81c8 100644 --- a/espnet2/asr/pit_espnet_model.py +++ b/espnet2/asr/pit_espnet_model.py @@ -29,6 +29,35 @@ def autocast(enabled=True): class PITLossWrapper(AbsLossWrapper): + """ + Permutation Invariant Training (PIT) Loss Wrapper. + + This class implements a wrapper for applying Permutation Invariant Training + to a given loss criterion function. It is designed to handle multi-speaker + scenarios where the order of speakers in the inference and reference may not + be consistent. + + Attributes: + criterion_fn (Callable): The loss criterion function to be wrapped. + num_ref (int): The number of reference speakers. + + Args: + criterion_fn (Callable): The loss criterion function to be wrapped. + num_ref (int): The number of reference speakers. + + Note: + This wrapper is similar to the PIT solver in espnet2/enh/loss/wrapper/pit_solver.py. + + Examples: + >>> criterion = torch.nn.MSELoss() + >>> pit_wrapper = PITLossWrapper(criterion, num_ref=2) + >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) + >>> ref = torch.randn(4, 2, 10) # (batch, num_speakers, features) + >>> inf_lens = torch.full((4, 2), 10) + >>> ref_lens = torch.full((4, 2), 10) + >>> loss, perm = pit_wrapper(inf, inf_lens, ref, ref_lens) + """ + def __init__(self, criterion_fn: Callable, num_ref: int): super().__init__() self.criterion_fn = criterion_fn @@ -42,15 +71,44 @@ def forward( ref_lens: torch.Tensor, others: Dict = None, ): - """PITLoss Wrapper function. Similar to espnet2/enh/loss/wrapper/pit_solver.py + """ + Apply Permutation Invariant Training (PIT) loss calculation. + + This method computes the PIT loss by finding the optimal permutation of + speakers that minimizes the total loss across all possible permutations. Args: - inf: Iterable[torch.Tensor], (batch, num_inf, ...) - inf_lens: Iterable[torch.Tensor], (batch, num_inf, ...) - ref: Iterable[torch.Tensor], (batch, num_ref, ...) - ref_lens: Iterable[torch.Tensor], (batch, num_ref, ...) - permute_inf: If true, permute the inference and inference_lens according to - the optimal permutation. + inf (torch.Tensor): Inferred output tensor. + Shape: (batch, num_inf, ...) + inf_lens (torch.Tensor): Lengths of inferred outputs. + Shape: (batch, num_inf) + ref (torch.Tensor): Reference tensor. + Shape: (batch, num_ref, ...) + ref_lens (torch.Tensor): Lengths of reference outputs. + Shape: (batch, num_ref) + others (Dict, optional): Additional arguments to be passed to the criterion function. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - The mean of the minimum losses across the batch (scalar). + - The optimal permutation for each item in the batch. + Shape: (batch_size, num_ref) + + Raises: + AssertionError: If the shapes of input tensors are inconsistent or + if num_ref doesn't match the number of speakers in the inputs. + + Note: + This method assumes that the number of inferred speakers (num_inf) is equal + to the number of reference speakers (num_ref). + + Examples: + >>> pit_wrapper = PITLossWrapper(criterion_fn, num_ref=2) + >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) + >>> ref = torch.randn(4, 2, 10) # (batch, num_speakers, features) + >>> inf_lens = torch.full((4, 2), 10) + >>> ref_lens = torch.full((4, 2), 10) + >>> loss, perm = pit_wrapper.forward(inf, inf_lens, ref, ref_lens) """ assert ( self.num_ref @@ -99,6 +157,37 @@ def pair_loss(permutation): @classmethod def permutate(self, perm, *args): + """ + Permute the input tensors according to the given permutation. + + This class method applies the optimal permutation to the input tensors, + rearranging the speaker order for each item in the batch. + + Args: + perm (torch.Tensor): The permutation tensor to apply. + Shape: (batch_size, num_ref) + *args: Variable length argument list of tensors to be permuted. + Each tensor should have shape (batch, num_inf, ...) + + Returns: + List[torch.Tensor]: A list of permuted tensors, each with the same shape + as its corresponding input tensor. + + Raises: + AssertionError: If the batch size or number of speakers is inconsistent + across the input tensors. + + Note: + This method is typically used after finding the optimal permutation + with the forward method to reorder the input tensors accordingly. + + Examples: + >>> pit_wrapper = PITLossWrapper(criterion_fn, num_ref=2) + >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) + >>> inf_lens = torch.full((4, 2), 10) + >>> perm = torch.tensor([[0, 1], [1, 0], [0, 1], [1, 0]]) + >>> permuted_inf, permuted_inf_lens = PITLossWrapper.permutate(perm, inf, inf_lens) + """ ret = [] batch_size = None num_ref = None @@ -119,7 +208,80 @@ def permutate(self, perm, *args): class ESPnetASRModel(SingleESPnetASRModel): - """CTC-attention hybrid Encoder-Decoder model""" + """ + CTC-attention hybrid Encoder-Decoder model for multi-speaker ASR. + + This class extends the SingleESPnetASRModel to support multi-speaker automatic speech + recognition using Permutation Invariant Training (PIT). It combines CTC and + attention-based approaches for improved ASR performance in scenarios with multiple speakers. + + Attributes: + num_inf (int): Number of inferences (outputs) from the model. + num_ref (int): Number of references (ground truth sequences). + pit_ctc (PITLossWrapper): PIT wrapper for the CTC loss. + + Args: + vocab_size (int): Size of the vocabulary. + token_list (Union[Tuple[str, ...], List[str]]): List of tokens. + frontend (Optional[AbsFrontend]): Frontend processing module. + specaug (Optional[AbsSpecAug]): SpecAugment module. + normalize (Optional[AbsNormalize]): Normalization module. + preencoder (Optional[AbsPreEncoder]): Pre-encoder module. + encoder (AbsEncoder): Encoder module. + postencoder (Optional[AbsPostEncoder]): Post-encoder module. + decoder (Optional[AbsDecoder]): Decoder module. + ctc (CTC): CTC module. + joint_network (Optional[torch.nn.Module]): Joint network for transducer-based models. + ctc_weight (float): Weight of CTC loss (0.0 < ctc_weight <= 1.0). + interctc_weight (float): Weight of intermediate CTC loss (must be 0.0 for multi-speaker ASR). + ignore_id (int): Padding value for the ignore index. + lsm_weight (float): Label smoothing weight. + length_normalized_loss (bool): Whether to normalize loss by length. + report_cer (bool): Whether to report Character Error Rate. + report_wer (bool): Whether to report Word Error Rate. + sym_space (str): Space symbol. + sym_blank (str): Blank symbol. + sym_sos (str): Start of sequence symbol. + sym_eos (str): End of sequence symbol. + extract_feats_in_collect_stats (bool): Whether to extract features in collect_stats. + lang_token_id (int): Language token ID. + num_inf (int): Number of inferences (outputs) from the model. + num_ref (int): Number of references (ground truth sequences). + + Note: + - This model requires that num_inf == num_ref. + - The interctc_weight must be set to 0.0 as intermediate CTC is not supported for multi-speaker ASR. + + Examples: + >>> model = ESPnetASRModel( + ... vocab_size=1000, + ... token_list=["", "", "a", "b", "c", ...], + ... frontend=frontend, + ... specaug=specaug, + ... normalize=normalize, + ... preencoder=preencoder, + ... encoder=encoder, + ... postencoder=postencoder, + ... decoder=decoder, + ... ctc=ctc, + ... joint_network=None, + ... ctc_weight=0.3, + ... interctc_weight=0.0, + ... ignore_id=-1, + ... lsm_weight=0.1, + ... length_normalized_loss=False, + ... report_cer=True, + ... report_wer=True, + ... sym_space="", + ... sym_blank="", + ... sym_sos="", + ... sym_eos="", + ... extract_feats_in_collect_stats=True, + ... lang_token_id=-1, + ... num_inf=2, + ... num_ref=2 + ... ) + """ @typechecked def __init__( @@ -199,14 +361,58 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss + """ + Forward pass of the ESPnetASRModel for multi-speaker ASR. + + This method processes the input speech and computes the loss for multi-speaker + automatic speech recognition using a combination of CTC and attention-based approaches. Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - kwargs: "utt_id" is among the input. + speech (torch.Tensor): Input speech tensor (Batch, Length, ...). + speech_lengths (torch.Tensor): Lengths of input speech sequences (Batch,). + text (torch.Tensor): Target text tensor for the first speaker (Batch, Length). + text_lengths (torch.Tensor): Lengths of target text sequences (Batch,). + **kwargs: Additional keyword arguments. + Expected to contain "text_spk{n}" and "text_spk{n}_lengths" for n = 2 to num_ref, + representing the text and text lengths for additional speakers. + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + - loss: The total loss for the batch. + - stats: A dictionary containing various statistics and metrics: + - loss: Total loss (detached). + - loss_att: Attention loss (if applicable, detached). + - loss_ctc: CTC loss (if applicable, detached). + - loss_transducer: Transducer loss (if applicable, detached). + - acc: Attention accuracy (if applicable). + - cer: Character Error Rate (if applicable). + - wer: Word Error Rate (if applicable). + - cer_ctc: Character Error Rate for CTC (if applicable). + - cer_transducer: Character Error Rate for Transducer (if applicable). + - wer_transducer: Word Error Rate for Transducer (if applicable). + - weight: Batch size (used for averaging). + + Raises: + AssertionError: If the input tensor dimensions are inconsistent or if the + batch sizes don't match across inputs. + + Note: + - This method handles both CTC and attention-based (and potentially transducer-based) ASR approaches. + - It uses Permutation Invariant Training (PIT) for handling multiple speakers. + - The method expects additional text inputs for each speaker beyond the first, + which should be provided in the kwargs as "text_spk{n}" and "text_spk{n}_lengths". + + Examples: + >>> speech = torch.randn(2, 1000, 80) # (batch, time, features) + >>> speech_lengths = torch.tensor([1000, 800]) + >>> text = torch.randint(0, 100, (2, 50)) # (batch, max_text_len) + >>> text_lengths = torch.tensor([45, 30]) + >>> text_spk2 = torch.randint(0, 100, (2, 40)) + >>> text_spk2_lengths = torch.tensor([35, 25]) + >>> loss, stats, weight = model.forward( + ... speech, speech_lengths, text, text_lengths, + ... text_spk2=text_spk2, text_spk2_lengths=text_spk2_lengths + ... ) """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified diff --git a/espnet2/asr/postencoder/abs_postencoder.py b/espnet2/asr/postencoder/abs_postencoder.py index cebfa3b7021..8ad063e13a6 100644 --- a/espnet2/asr/postencoder/abs_postencoder.py +++ b/espnet2/asr/postencoder/abs_postencoder.py @@ -5,12 +5,78 @@ class AbsPostEncoder(torch.nn.Module, ABC): + """ + Abstract base class for post-encoder modules in a neural network. + + This class defines the interface for post-encoder modules that process + encoded input sequences. It inherits from both torch.nn.Module and ABC + (Abstract Base Class). + + Attributes: + None + + Note: + Subclasses must implement the `output_size` and `forward` methods. + + Example: + class CustomPostEncoder(AbsPostEncoder): + def output_size(self) -> int: + return 256 + + def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + # Custom implementation + pass + """ + @abstractmethod def output_size(self) -> int: + """ + Returns the output size of the post-encoder. + + This abstract method should be implemented by subclasses to specify + the size of the output tensor produced by the post-encoder. + + Returns: + int: The size of the output tensor. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Example: + class CustomPostEncoder(AbsPostEncoder): + def output_size(self) -> int: + return 256 # Example output size + """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Processes the input tensor through the post-encoder. + + This abstract method should be implemented by subclasses to define the + forward pass of the post-encoder. + + Args: + input (torch.Tensor): The input tensor to be processed. + input_lengths (torch.Tensor): The lengths of each sequence in the input tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - The processed output tensor. + - The updated lengths of each sequence in the output tensor. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Example: + class CustomPostEncoder(AbsPostEncoder): + def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + # Example implementation + processed_output = some_processing(input) + updated_lengths = input_lengths # Or modify if needed + return processed_output, updated_lengths + """ raise NotImplementedError diff --git a/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py b/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py index 75e88a23688..4478911f634 100644 --- a/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py +++ b/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py @@ -24,7 +24,33 @@ class HuggingFaceTransformersPostEncoder(AbsPostEncoder): - """Hugging Face Transformers PostEncoder.""" + """ + Hugging Face Transformers PostEncoder. + + This class implements a post-encoder using Hugging Face Transformers models. + It can be used to process the output of an encoder in a speech recognition + or other sequence processing task. + + Attributes: + transformer (transformers.PreTrainedModel): The Hugging Face transformer model. + lang_token_embed (torch.Tensor): Language token embedding, if applicable. + linear_in (torch.nn.Linear): Linear layer to project input to transformer dimension. + length_adaptor (torch.nn.Sequential): Sequence of layers for length adaptation. + length_adaptor_ratio (int): Ratio of length reduction in the adaptor. + + Args: + input_size (int): Size of the input features. + model_name_or_path (str): Name or path of the pre-trained Hugging Face model. + length_adaptor_n_layers (int, optional): Number of layers in the length adaptor. Defaults to 0. + lang_token_id (int, optional): ID of the language token. Defaults to -1. + + Raises: + ImportError: If the 'transformers' library is not installed. + + Note: + This class requires the 'transformers' library to be installed. + The length adaptor is implemented as described in https://aclanthology.org/2021.acl-long.68.pdf + """ @typechecked def __init__( @@ -127,7 +153,36 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward.""" + """ + Forward pass of the HuggingFaceTransformersPostEncoder. + + This method processes the input through the length adaptor, linear projection, + and the Hugging Face transformer model. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, time, feature_dim). + input_lengths (torch.Tensor): Tensor of input lengths for each sequence in the batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - output (torch.Tensor): The output of the transformer model. + - input_lengths (torch.Tensor): Updated lengths after processing. + + Raises: + TooShortUttError: If the input sequence is too short for subsampling. + + Note: + - The input is first processed by the length adaptor, which may reduce its temporal dimension. + - If a language token is specified, it's prepended to the input. + - The method handles different configurations of transformer models, including + encoder-decoder models, XLNet, T5, and GPT-2. + + Examples: + >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") + >>> input_tensor = torch.randn(32, 100, 256) # (batch_size, time, feature_dim) + >>> input_lengths = torch.full((32,), 100) + >>> output, output_lengths = encoder.forward(input_tensor, input_lengths) + """ if input.size(1) < self.length_adaptor_ratio: raise TooShortUttError( f"has {input.size(1)} frames and is too short for subsampling " @@ -178,11 +233,40 @@ def forward( return output, input_lengths def reload_pretrained_parameters(self): - self.transformer.load_state_dict(self.pretrained_params) + """ + Reload the pretrained parameters of the transformer model. + + This method resets the transformer's parameters to their original pretrained values. + It's useful for resetting the model to its initial state, especially after fine-tuning. + + Note: + This method logs an info message upon successful reloading of parameters. + + Examples: + >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") + >>> # After some training or parameter updates + >>> encoder.reload_pretrained_parameters() + # This will reset the transformer's parameters to their original pretrained values + """ logging.info("Pretrained Transformers model parameters reloaded!") def output_size(self) -> int: - """Get the output size.""" + """ + Get the output size of the transformer model. + + Returns: + int: The size of the output features from the transformer model. + + Note: + This method returns the hidden size of the transformer model, which + corresponds to the dimensionality of the output features. + + Examples: + >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") + >>> output_dim = encoder.output_size() + >>> print(output_dim) + 768 # For BERT base model + """ return self.transformer.config.hidden_size diff --git a/espnet2/asr/postencoder/length_adaptor_postencoder.py b/espnet2/asr/postencoder/length_adaptor_postencoder.py index 40420197c60..1658d0dd5aa 100644 --- a/espnet2/asr/postencoder/length_adaptor_postencoder.py +++ b/espnet2/asr/postencoder/length_adaptor_postencoder.py @@ -14,7 +14,24 @@ class LengthAdaptorPostEncoder(AbsPostEncoder): - """Length Adaptor PostEncoder.""" + """ + Length Adaptor PostEncoder for adjusting the length of input sequences. + + This class implements a Length Adaptor PostEncoder, which is designed to modify + the length of input sequences through a series of convolutional layers. It can + optionally include an initial linear layer for input embedding. + + Attributes: + embed (torch.nn.Sequential or None): Optional input embedding layer. + length_adaptor (torch.nn.Sequential): Sequence of convolutional layers for length adaptation. + length_adaptor_ratio (int): The ratio by which the input length is reduced. + return_int_enc (bool): Flag to determine if intermediate encodings should be returned. + out_sz (int): The output size of the encoder. + + Note: + This implementation is based on the Length Adaptor described in + https://aclanthology.org/2021.acl-long.68.pdf + """ @typechecked def __init__( @@ -59,7 +76,38 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward.""" + """ + Forward pass of the Length Adaptor PostEncoder. + + This method processes the input tensor through the length adaptor layers, + adjusting its length according to the configured length_adaptor_ratio. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, time_steps, features). + input_lengths (torch.Tensor): Tensor containing the lengths of each sequence in the batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - output tensor of shape (batch_size, adjusted_time_steps, features) + - tensor of adjusted input lengths + + Raises: + TooShortUttError: If the input sequence is too short for the configured subsampling. + + Examples: + >>> encoder = LengthAdaptorPostEncoder(input_size=256, length_adaptor_n_layers=2) + >>> input_tensor = torch.randn(32, 100, 256) + >>> input_lengths = torch.full((32,), 100) + >>> output, output_lengths = encoder(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 25, 256]) + >>> print(output_lengths) + tensor([25, 25, 25, ..., 25, 25, 25]) + + Note: + The input tensor is permuted before and after passing through the length adaptor + to match the expected input format of Conv1d layers. + """ if input.size(1) < self.length_adaptor_ratio: raise TooShortUttError( f"has {input.size(1)} frames and is too short for subsampling " @@ -83,5 +131,25 @@ def forward( return input, input_lengths def output_size(self) -> int: - """Get the output size.""" + """ + Get the output size of the Length Adaptor PostEncoder. + + Returns: + int: The size of the output features. + + Note: + This method returns the value stored in the `out_sz` attribute, which is set + during the initialization of the LengthAdaptorPostEncoder. It represents + either the original input size or the size after the optional linear embedding, + depending on the configuration. + + Examples: + >>> encoder = LengthAdaptorPostEncoder(input_size=256, output_size=512) + >>> print(encoder.output_size()) + 512 + + >>> encoder = LengthAdaptorPostEncoder(input_size=256) + >>> print(encoder.output_size()) + 256 + """ return self.out_sz diff --git a/espnet2/asr/preencoder/abs_preencoder.py b/espnet2/asr/preencoder/abs_preencoder.py index 67777477e0b..27c5097db57 100644 --- a/espnet2/asr/preencoder/abs_preencoder.py +++ b/espnet2/asr/preencoder/abs_preencoder.py @@ -5,12 +5,95 @@ class AbsPreEncoder(torch.nn.Module, ABC): + """ + Abstract base class for pre-encoder modules in a neural network. + + This class defines the interface for pre-encoder modules, which are typically + used to process input data before it's passed to the main encoder in a + neural network architecture. + + Attributes: + None + + Examples: + ```python + class MyPreEncoder(AbsPreEncoder): + def output_size(self) -> int: + return 256 + + def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + # Implementation of forward pass + pass + + pre_encoder = MyPreEncoder() + ``` + + Note: + Subclasses must implement the `output_size` and `forward` methods. + """ + @abstractmethod def output_size(self) -> int: + """ + Returns the output size of the pre-encoder. + + This method should be implemented by subclasses to specify the + dimensionality of the output produced by the pre-encoder. + + Returns: + int: The size of the output tensor produced by the pre-encoder. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Examples: + ```python + class MyPreEncoder(AbsPreEncoder): + def output_size(self) -> int: + return 256 + + pre_encoder = MyPreEncoder() + output_dim = pre_encoder.output_size() # Returns 256 + ``` + """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Performs the forward pass of the pre-encoder. + + This method should be implemented by subclasses to define the forward + computation of the pre-encoder. + + Args: + input (torch.Tensor): The input tensor to be processed. + input_lengths (torch.Tensor): A tensor containing the lengths of each + sequence in the input batch. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - The output tensor after pre-encoding. + - A tensor of output lengths corresponding to each sequence in the batch. + + Raises: + NotImplementedError: If the method is not implemented by a subclass. + + Examples: + ```python + class MyPreEncoder(AbsPreEncoder): + def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + # Example implementation + output = self.some_processing(input) + output_lengths = input_lengths # Assuming no change in sequence length + return output, output_lengths + + pre_encoder = MyPreEncoder() + input_tensor = torch.randn(32, 100, 80) # Batch size 32, 100 time steps, 80 features + input_lengths = torch.full((32,), 100) # All sequences in batch have length 100 + output, output_lengths = pre_encoder(input_tensor, input_lengths) + ``` + """ raise NotImplementedError diff --git a/espnet2/asr/preencoder/linear.py b/espnet2/asr/preencoder/linear.py index 94e857bdb12..2c56d6720ba 100644 --- a/espnet2/asr/preencoder/linear.py +++ b/espnet2/asr/preencoder/linear.py @@ -13,7 +13,35 @@ class LinearProjection(AbsPreEncoder): - """Linear Projection Preencoder.""" + """ + Linear Projection Preencoder. + + This class implements a linear projection preencoder that applies a linear transformation + followed by dropout to the input features. It is a subclass of AbsPreEncoder. + + Attributes: + output_dim (int): The dimension of the output features. + linear_out (torch.nn.Linear): The linear transformation layer. + dropout (torch.nn.Dropout): The dropout layer. + + Args: + input_size (int): The size of the input features. + output_size (int): The size of the output features. + dropout (float, optional): The dropout rate. Defaults to 0.0. + + Example: + >>> import torch + >>> preencoder = LinearProjection(input_size=100, output_size=80, dropout=0.1) + >>> input_tensor = torch.randn(32, 10, 100) # (batch_size, sequence_length, input_size) + >>> input_lengths = torch.full((32,), 10) + >>> output, output_lengths = preencoder(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 10, 80]) + + Note: + This preencoder does not modify the input lengths, so the output_lengths + will be the same as the input_lengths. + """ @typechecked def __init__(self, input_size: int, output_size: int, dropout: float = 0.0): @@ -27,10 +55,52 @@ def __init__(self, input_size: int, output_size: int, dropout: float = 0.0): def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward.""" + """ + Forward pass of the LinearProjection preencoder. + + This method applies dropout to the input, then performs a linear transformation. + + Args: + input (torch.Tensor): Input tensor of shape (batch_size, sequence_length, input_size). + input_lengths (torch.Tensor): Tensor of input sequence lengths of shape (batch_size,). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - output (torch.Tensor): The transformed output tensor of shape + (batch_size, sequence_length, output_size). + - input_lengths (torch.Tensor): The input sequence lengths, unchanged. + + Note: + This method does not modify the input lengths, so the returned input_lengths + are the same as the input. + + Example: + >>> preencoder = LinearProjection(input_size=100, output_size=80) + >>> input_tensor = torch.randn(32, 10, 100) + >>> input_lengths = torch.full((32,), 10) + >>> output, output_lengths = preencoder.forward(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 10, 80]) + >>> print(torch.all(output_lengths == input_lengths)) + True + """ output = self.linear_out(self.dropout(input)) return output, input_lengths # no state in this layer def output_size(self) -> int: - """Get the output size.""" + """ + Get the output size of the LinearProjection preencoder. + + Returns: + int: The dimension of the output features. + + Example: + >>> preencoder = LinearProjection(input_size=100, output_size=80) + >>> print(preencoder.output_size()) + 80 + + Note: + This method returns the value of the `output_dim` attribute, which is set + during the initialization of the LinearProjection instance. + """ return self.output_dim diff --git a/espnet2/asr/preencoder/sinc.py b/espnet2/asr/preencoder/sinc.py index 778ccf8acfe..7e0da592002 100644 --- a/espnet2/asr/preencoder/sinc.py +++ b/espnet2/asr/preencoder/sinc.py @@ -16,26 +16,37 @@ class LightweightSincConvs(AbsPreEncoder): - """Lightweight Sinc Convolutions. + """ + Lightweight Sinc Convolutions for end-to-end speech recognition from raw audio. - Instead of using precomputed features, end-to-end speech recognition - can also be done directly from raw audio using sinc convolutions, as - described in "Lightweight End-to-End Speech Recognition from Raw Audio - Data Using Sinc-Convolutions" by Kürzinger et al. - https://arxiv.org/abs/2010.07597 + This class implements the Lightweight Sinc Convolutions as described in + "Lightweight End-to-End Speech Recognition from Raw Audio Data Using + Sinc-Convolutions" by Kürzinger et al. It processes raw audio input + instead of using precomputed features, serving as a pre-encoder in the + speech recognition pipeline. - To use Sinc convolutions in your model instead of the default f-bank - frontend, set this module as your pre-encoder with `preencoder: sinc` - and use the input of the sliding window frontend with - `frontend: sliding_window` in your yaml configuration file. - So that the process flow is: + To use this module, set it as the pre-encoder with `preencoder: sinc` + and use the sliding window frontend with `frontend: sliding_window` in your + YAML configuration file. + The process flow should be: Frontend (SlidingWindow) -> SpecAug -> Normalization -> Pre-encoder (LightweightSincConvs) -> Encoder -> Decoder - Note that this method also performs data augmentation in time domain - (vs. in spectral domain in the default frontend). - Use `plot_sinc_filters.py` to visualize the learned Sinc filters. + This method performs data augmentation in the time domain, as opposed to + the spectral domain in the default frontend. + + Attributes: + fs (int): Sample rate of the input audio. + in_channels (int): Number of input channels. + out_channels (int): Number of output channels (for each input channel). + activation_type (str): Type of activation function used. + dropout_type (str): Type of dropout function used. + windowing_type (str): Type of windowing function used. + scale_type (str): Type of filter-bank initialization scale. + + Note: + Use `plot_sinc_filters.py` to visualize the learned Sinc filters. """ @typechecked @@ -166,24 +177,30 @@ def gen_lsc_block( dropout_probability: float = 0.15, avgpool=False, ): - """Generate a convolutional block for Lightweight Sinc convolutions. + """ + Generate a convolutional block for Lightweight Sinc convolutions. Each block consists of either a depthwise or a depthwise-separable - convolutions together with dropout, (batch-)normalization layer, and + convolution together with dropout, (batch-)normalization layer, and an optional average-pooling layer. Args: - in_channels: Number of input channels. - out_channels: Number of output channels. - depthwise_kernel_size: Kernel size of the depthwise convolution. - depthwise_stride: Stride of the depthwise convolution. - depthwise_groups: Number of groups of the depthwise convolution. - pointwise_groups: Number of groups of the pointwise convolution. - dropout_probability: Dropout probability in the block. - avgpool: If True, an AvgPool layer is inserted. + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + depthwise_kernel_size (int): Kernel size of the depthwise convolution. Defaults to 9. + depthwise_stride (int): Stride of the depthwise convolution. Defaults to 1. + depthwise_groups (int, optional): Number of groups of the depthwise convolution. + If None, it's set to the GCD of in_channels and out_channels. + pointwise_groups (int): Number of groups of the pointwise convolution. Defaults to 0. + dropout_probability (float): Dropout probability in the block. Defaults to 0.15. + avgpool (bool): If True, an AvgPool layer is inserted. Defaults to False. Returns: - torch.nn.Sequential: Neural network building block. + torch.nn.Sequential: Neural network building block for Lightweight Sinc convolutions. + + Note: + The block structure adapts based on the provided parameters, allowing for + flexible configuration of the convolutional layers. """ block = OrderedDict() if not depthwise_groups: @@ -210,7 +227,21 @@ def gen_lsc_block( return torch.nn.Sequential(block) def espnet_initialization_fn(self): - """Initialize sinc filters with filterbank values.""" + """ + Initialize sinc filters and batch normalization layers. + + This method initializes the sinc filters with filterbank values and sets + the initial weights and biases for batch normalization layers in the network. + + The initialization process includes: + 1. Initializing the sinc filters using the `init_filters` method. + 2. Setting the weight data of all BatchNorm1d layers to 1.0. + 3. Setting the bias data of all BatchNorm1d layers to 0.0. + + Note: + This initialization is specific to the Lightweight Sinc Convolutions + architecture and helps in achieving better initial performance. + """ self.filters.init_filters() for block in self.blocks: for layer in block: @@ -221,18 +252,35 @@ def espnet_initialization_fn(self): def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply Lightweight Sinc Convolutions. - - The input shall be formatted as (B, T, C_in, D_in) - with B as batch size, T as time dimension, C_in as channels, - and D_in as feature dimension. + """ + Apply Lightweight Sinc Convolutions to the input tensor. - The output will then be (B, T, C_out*D_out) - with C_out and D_out as output dimensions. + Args: + input (torch.Tensor): Input tensor of shape (B, T, C_in, D_in), where: + B is the batch size, + T is the time dimension, + C_in is the number of input channels, + D_in is the input feature dimension. + input_lengths (torch.Tensor): Tensor containing the lengths of each sequence in the batch. - The current module structure only handles D_in=400, so that D_out=1. - Remark for the multichannel case: C_out is the number of out_channels - given at initialization multiplied with C_in. + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - output_frames (torch.Tensor): Output tensor of shape (B, T, C_out*D_out), where: + C_out is the number of output channels, + D_out is the output feature dimension. + - input_lengths (torch.Tensor): The input lengths tensor (unchanged). + + Note: + The current module structure only handles D_in=400, resulting in D_out=1. + For the multichannel case, C_out is the number of out_channels given at + initialization multiplied by C_in. + + Example: + >>> input_tensor = torch.randn(32, 1000, 1, 400) # (B, T, C_in, D_in) + >>> input_lengths = torch.full((32,), 1000) + >>> output, output_lengths = self.forward(input_tensor, input_lengths) + >>> print(output.shape) + torch.Size([32, 1000, 256]) # Assuming out_channels=256 """ # Transform input data: # (B, T, C_in, D_in) -> (B*T, C_in, D_in) @@ -246,14 +294,37 @@ def forward( return output_frames, input_lengths # no state in this layer def output_size(self) -> int: - """Get the output size.""" + """ + Get the output size of the Lightweight Sinc Convolutions. + + Returns: + int: The total number of output features, calculated as the product + of the number of output channels and the number of input channels. + + Note: + This method is useful for determining the size of the output tensor + produced by the Lightweight Sinc Convolutions, which can be helpful + when connecting this module to subsequent layers in the network. + """ return self.out_channels * self.in_channels class SpatialDropout(torch.nn.Module): - """Spatial dropout module. + """ + Spatial dropout module for applying dropout to full channels. + + This module applies dropout to entire channels on tensors of input shape (B, C, D), + where B is the batch size, C is the number of channels, and D is the feature dimension. + It's designed to maintain spatial coherence by dropping out entire feature maps + instead of individual elements. - Apply dropout to full channels on tensors of input (B, C, D) + Attributes: + dropout (torch.nn.Dropout2d): The underlying dropout layer. + shape (tuple): The shape used for permuting the input tensor. + + Note: + This implementation is particularly useful for 1D convolutional neural networks + where you want to drop entire channels rather than random elements. """ @typechecked @@ -275,7 +346,25 @@ def __init__( self.shape = (shape,) def forward(self, x: torch.Tensor) -> torch.Tensor: - """Forward of spatial dropout module.""" + """ + Apply spatial dropout to the input tensor. + + This method applies dropout to entire channels of the input tensor. + It first permutes the input tensor according to the specified shape, + applies dropout, and then permutes the tensor back to its original shape. + + Args: + x (torch.Tensor): Input tensor of shape (B, C, D), where B is the batch size, + C is the number of channels, and D is the feature dimension. + + Returns: + torch.Tensor: Output tensor with spatial dropout applied, maintaining the + same shape as the input tensor. + + Note: + The dropout is applied to entire channels, preserving spatial coherence + within each channel. + """ y = x.permute(*self.shape) y = self.dropout(y) return y.permute(*self.shape) diff --git a/espnet2/asr/specaug/abs_specaug.py b/espnet2/asr/specaug/abs_specaug.py index 6c9c6d8ea18..95e3df0d309 100644 --- a/espnet2/asr/specaug/abs_specaug.py +++ b/espnet2/asr/specaug/abs_specaug.py @@ -4,14 +4,62 @@ class AbsSpecAug(torch.nn.Module): - """Abstract class for the augmentation of spectrogram + """ + Abstract base class for spectrogram augmentation. + + This class defines the interface for spectrogram augmentation modules in a + neural network pipeline. Spectrogram augmentation is typically applied after + the frontend processing and before normalization, encoding, and decoding steps. - The process-flow: + Attributes: + None - Frontend -> SpecAug -> Normalization -> Encoder -> Decoder + Note: + Subclasses must implement the `forward` method to define specific + augmentation techniques. + + Examples: + >>> class MySpecAug(AbsSpecAug): + ... def forward(self, x, x_lengths=None): + ... # Implement augmentation logic here + ... return augmented_x, x_lengths + ... + >>> spec_aug = MySpecAug() + >>> augmented_spec, lengths = spec_aug(input_spec, input_lengths) """ def forward( self, x: torch.Tensor, x_lengths: torch.Tensor = None ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Apply spectrogram augmentation to the input tensor. + + This method should be implemented by subclasses to perform specific + augmentation techniques on the input spectrogram. + + Args: + x (torch.Tensor): Input spectrogram tensor. + x_lengths (torch.Tensor, optional): Tensor containing the lengths of each + sequence in the batch. Defaults to None. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - The augmented spectrogram tensor. + - The updated lengths tensor (if x_lengths was provided, otherwise None). + + Raises: + NotImplementedError: This method must be implemented by subclasses. + + Examples: + >>> class MySpecAug(AbsSpecAug): + ... def forward(self, x, x_lengths=None): + ... # Apply augmentation (e.g., time warping, frequency masking) + ... augmented_x = some_augmentation_function(x) + ... return augmented_x, x_lengths + ... + >>> spec_aug = MySpecAug() + >>> input_spec = torch.randn(32, 80, 100) # (batch_size, num_mels, time_steps) + >>> input_lengths = torch.full((32,), 100) + >>> augmented_spec, updated_lengths = spec_aug(input_spec, input_lengths) + """ raise NotImplementedError diff --git a/espnet2/asr/specaug/specaug.py b/espnet2/asr/specaug/specaug.py index f35d1468585..193af8a7904 100644 --- a/espnet2/asr/specaug/specaug.py +++ b/espnet2/asr/specaug/specaug.py @@ -8,17 +8,44 @@ class SpecAug(AbsSpecAug): - """Implementation of SpecAug. + """ + Implementation of SpecAugment for speech augmentation. + + This class applies various augmentation techniques to speech spectrograms, + including time warping, frequency masking, and time masking. + + Attributes: + apply_time_warp (bool): Whether to apply time warping. + apply_freq_mask (bool): Whether to apply frequency masking. + apply_time_mask (bool): Whether to apply time masking. + time_warp (TimeWarp): Time warping object if enabled, else None. + freq_mask (MaskAlongAxis): Frequency masking object if enabled, else None. + time_mask (Union[MaskAlongAxis, MaskAlongAxisVariableMaxWidth]): Time masking object if enabled, else None. + + Args: + apply_time_warp (bool): Whether to apply time warping. Defaults to True. + time_warp_window (int): Window size for time warping. Defaults to 5. + time_warp_mode (str): Interpolation mode for time warping. Defaults to "bicubic". + apply_freq_mask (bool): Whether to apply frequency masking. Defaults to True. + freq_mask_width_range (Union[int, Sequence[int]]): Range of width for frequency masks. Defaults to (0, 20). + num_freq_mask (int): Number of frequency masks to apply. Defaults to 2. + apply_time_mask (bool): Whether to apply time masking. Defaults to True. + time_mask_width_range (Optional[Union[int, Sequence[int]]]): Range of width for time masks. Defaults to None. + time_mask_width_ratio_range (Optional[Union[float, Sequence[float]]]): Range of width ratio for time masks. Defaults to None. + num_time_mask (int): Number of time masks to apply. Defaults to 2. - Reference: - Daniel S. Park et al. - "SpecAugment: A Simple Data - Augmentation Method for Automatic Speech Recognition" + Raises: + ValueError: If no augmentation technique is applied or if both time_mask_width_range + and time_mask_width_ratio_range are specified. - .. warning:: - When using cuda mode, time_warp doesn't have reproducibility - due to `torch.nn.functional.interpolate`. + Note: + This implementation is based on the SpecAugment paper by Daniel S. Park et al. + When using CUDA mode, time warping may not be reproducible due to + `torch.nn.functional.interpolate`. + Examples: + >>> specaug = SpecAug(apply_time_warp=True, apply_freq_mask=True, apply_time_mask=True) + >>> augmented_spec, lengths = specaug(input_spec, input_lengths) """ def __init__( @@ -88,6 +115,27 @@ def __init__( self.time_mask = None def forward(self, x, x_lengths=None): + """ + Apply SpecAugment to the input spectrogram. + + This method applies the configured augmentation techniques (time warping, + frequency masking, and time masking) to the input spectrogram. + + Args: + x (Tensor): Input spectrogram tensor of shape (batch_size, num_channels, num_freq, num_time). + x_lengths (Tensor, optional): Lengths of each sequence in the batch. Defaults to None. + + Returns: + Tuple[Tensor, Tensor]: + - Augmented spectrogram tensor of the same shape as the input. + - Updated lengths tensor (if x_lengths was provided, otherwise None). + + Examples: + >>> specaug = SpecAug() + >>> input_spec = torch.randn(32, 1, 80, 100) # (batch_size, channels, freq, time) + >>> input_lengths = torch.full((32,), 100) + >>> augmented_spec, augmented_lengths = specaug.forward(input_spec, input_lengths) + """ if self.time_warp is not None: x, x_lengths = self.time_warp(x, x_lengths) if self.freq_mask is not None: diff --git a/espnet2/asr/state_spaces/attention.py b/espnet2/asr/state_spaces/attention.py index 70f9579b4ac..68ce97b9dd7 100644 --- a/espnet2/asr/state_spaces/attention.py +++ b/espnet2/asr/state_spaces/attention.py @@ -10,16 +10,43 @@ class MultiHeadedAttention(SequenceModule): - """Multi-Head Attention layer inheriting SequenceModule. - - Comparing default MHA module in ESPnet, this module returns additional dummy state - and has step function for autoregressive inference. + """ + Multi-Head Attention layer inheriting from SequenceModule. + + This class implements a Multi-Head Attention mechanism, which allows the model + to jointly attend to information from different representation subspaces at + different positions. It extends the functionality of the default Multi-Head + Attention module in ESPnet by returning an additional dummy state and + providing a step function for autoregressive inference. + + Attributes: + d_k (int): The dimension of the key and query vectors. + h (int): The number of attention heads. + linear_q (nn.Linear): Linear transformation for queries. + linear_k (nn.Linear): Linear transformation for keys. + linear_v (nn.Linear): Linear transformation for values. + linear_out (nn.Linear): Linear transformation for output. + attn (torch.Tensor): Attention weights. + dropout (nn.Dropout): Dropout layer. + d_output (int): The output dimension. Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - + n_feat (int): The number of expected features in the input. + n_head (int): The number of attention heads. + dropout (float, optional): Dropout probability. Defaults to 0.0. + transposed (bool, optional): If True, expects transposed input. Defaults to False. + **kwargs: Additional keyword arguments. + + Note: + This implementation assumes that the dimension of the key (d_k) always + equals the dimension of the value (d_v), and that n_feat is divisible by n_head. + + Example: + >>> mha = MultiHeadedAttention(n_feat=512, n_head=8, dropout=0.1) + >>> query = torch.randn(32, 10, 512) # (batch_size, seq_len, n_feat) + >>> output, _ = mha(query) + >>> output.shape + torch.Size([32, 10, 512]) """ def __init__(self, n_feat, n_head, dropout=0.0, transposed=False, **kwargs): @@ -38,18 +65,36 @@ def __init__(self, n_feat, n_head, dropout=0.0, transposed=False, **kwargs): self.d_output = n_feat def forward_qkv(self, query, key, value): - """Transform query, key and value. + """ + Transform query, key, and value tensors for multi-head attention. + + This method applies linear transformations to the input query, key, and value + tensors, reshapes them to separate the head dimension, and transposes the + resulting tensors to prepare them for the attention mechanism. Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). + query (torch.Tensor): Query tensor of shape (batch_size, time1, size). + key (torch.Tensor): Key tensor of shape (batch_size, time2, size). + value (torch.Tensor): Value tensor of shape (batch_size, time2, size). Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - + tuple: A tuple containing: + - q (torch.Tensor): Transformed query tensor of shape (batch_size, n_head, time1, d_k). + - k (torch.Tensor): Transformed key tensor of shape (batch_size, n_head, time2, d_k). + - v (torch.Tensor): Transformed value tensor of shape (batch_size, n_head, time2, d_k). + + Note: + The input tensors are expected to have the same size in their last dimension, + which should be equal to n_feat (the number of features) in the MultiHeadedAttention class. + + Example: + >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) + >>> query = torch.randn(32, 10, 512) + >>> key = torch.randn(32, 15, 512) + >>> value = torch.randn(32, 15, 512) + >>> q, k, v = mha.forward_qkv(query, key, value) + >>> q.shape, k.shape, v.shape + (torch.Size([32, 8, 10, 64]), torch.Size([32, 8, 15, 64]), torch.Size([32, 8, 15, 64])) """ n_batch = query.size(0) q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) @@ -62,17 +107,37 @@ def forward_qkv(self, query, key, value): return q, k, v def forward_attention(self, value, scores, mask): - """Compute attention context vector. + """ + Compute attention context vector. + + This method applies the attention mechanism to the transformed value tensor + using the provided attention scores and mask. It computes the weighted sum + of values based on the attention distribution. Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). + value (torch.Tensor): Transformed value tensor of shape (batch_size, n_head, time2, d_k). + scores (torch.Tensor): Attention score tensor of shape (batch_size, n_head, time1, time2). + mask (torch.Tensor): Mask tensor of shape (batch_size, 1, time2) or (batch_size, time1, time2). + If provided, positions with 1 are masked (i.e., set to -inf before softmax). Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - + torch.Tensor: Output tensor of shape (batch_size, time1, d_model) containing + the weighted sum of values according to the attention distribution. + + Note: + - The method updates the `self.attn` attribute with the computed attention weights. + - If a mask is provided, it is applied before the softmax to prevent attention + to certain positions. + - Dropout is applied to the attention weights before computing the weighted sum. + + Example: + >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) + >>> value = torch.randn(32, 8, 15, 64) + >>> scores = torch.randn(32, 8, 10, 15) + >>> mask = torch.ones(32, 1, 15).bool() + >>> output = mha.forward_attention(value, scores, mask) + >>> output.shape + torch.Size([32, 10, 512]) """ n_batch = value.size(0) if mask is not None: @@ -96,18 +161,41 @@ def forward_attention(self, value, scores, mask): return self.linear_out(x) # (batch, time1, d_model) def forward(self, query, memory=None, mask=None, *args, **kwargs): - """Compute scaled dot product attention. + """ + Compute scaled dot product attention. + + This method performs the full multi-head attention operation, including the + transformation of inputs, computation of attention scores, and application + of attention weights to produce the final output. Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). + query (torch.Tensor): Query tensor of shape (batch_size, time1, size). + memory (torch.Tensor, optional): Memory tensor of shape (batch_size, time2, size). + If None, self-attention is performed using the query as both key and value. + mask (torch.Tensor, optional): Mask tensor of shape (batch_size, 1, time2) or + (batch_size, time1, time2). Positions with 1 are masked. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - + tuple: A tuple containing: + - torch.Tensor: Output tensor of shape (batch_size, time1, d_model). + - None: A placeholder for compatibility with other modules that return a state. + + Note: + - If memory is None, self-attention is performed using the query as both key and value. + - The attention scores are scaled by 1/sqrt(d_k) before applying the mask and softmax. + - The output is obtained by applying the attention weights to the value vectors + and then passing through a final linear transformation. + + Example: + >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) + >>> query = torch.randn(32, 10, 512) + >>> memory = torch.randn(32, 15, 512) + >>> mask = torch.ones(32, 1, 15).bool() + >>> output, _ = mha.forward(query, memory, mask) + >>> output.shape + torch.Size([32, 10, 512]) """ # self-attention if memory is None: @@ -117,6 +205,42 @@ def forward(self, query, memory=None, mask=None, *args, **kwargs): return self.forward_attention(v, scores, mask), None def step(self, query, state, memory=None, mask=None, **kwargs): + """ + Perform a single step of multi-head attention for autoregressive inference. + + This method is designed for use in autoregressive decoding, where attention is + computed for a single time step. It wraps the forward method to handle the + single-step case efficiently. + + Args: + query (torch.Tensor): Query tensor for the current step, shape (batch_size, 1, size). + state (Any): The previous state (unused in this implementation). + memory (torch.Tensor, optional): Memory tensor of shape (batch_size, time2, size). + If None, self-attention is performed using the query as both key and value. + mask (torch.Tensor, optional): Mask tensor of shape (batch_size, 1, time2) or + (batch_size, 1, time2). Positions with 1 are masked. + **kwargs: Additional keyword arguments passed to the forward method. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor for the current step, shape (batch_size, size). + - Any: The updated state (same as input state in this implementation). + + Note: + - This method assumes that the query is for a single time step. + - The output is squeezed to remove the time dimension, as it's always 1 in this case. + - The state is passed through unchanged, as the current implementation + doesn't utilize persistent state between steps. + + Example: + >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) + >>> query = torch.randn(32, 1, 512) + >>> memory = torch.randn(32, 15, 512) + >>> mask = torch.ones(32, 1, 15).bool() + >>> output, new_state = mha.step(query, None, memory, mask) + >>> output.shape + torch.Size([32, 512]) + """ if memory is None: memory = query return self.forward(query, memory, mask=mask, **kwargs)[0].squeeze(1), state diff --git a/espnet2/asr/state_spaces/base.py b/espnet2/asr/state_spaces/base.py index 91513920ead..92141887577 100644 --- a/espnet2/asr/state_spaces/base.py +++ b/espnet2/asr/state_spaces/base.py @@ -6,32 +6,84 @@ class SequenceModule(nn.Module): - """Abstract sequence model class. + """ + Abstract base class for sequence models. + + This class defines the interface for sequence models that transform input tensors + of shape (n_batch, l_sequence, d_model) to (n_batch, l_sequence, d_output). + + All models inheriting from this class must implement the required methods and + attributes. Optional methods provide additional functionality for recurrent + processing and state management. + + Attributes: + d_model (int): Model dimension (generally same as input dimension). + d_output (int): Output dimension of the model. + + Args: + d_model (int): The input dimension of the model. + transposed (bool, optional): If True, expects input in (n_batch, d_model, l_sequence) + format. Defaults to False. + + Required Methods: + forward(self, x, state=None, **kwargs): Performs the forward pass of the model. + __init__(self, d_model, transposed=False, **kwargs): Initializes the model. - All models must adhere to this interface + Optional Methods: + default_state(self, *batch_shape, device=None): Creates initial state for a batch. + step(self, x, state=None, **kwargs): Processes one step of the input sequence. + state_to_tensor(self): Returns a function to map state to a single tensor. - A SequenceModule is generally a model that transforms an input of shape - (n_batch, l_sequence, d_model) to (n_batch, l_sequence, d_output) + Properties: + d_state (int): Dimension of the output of self.state_to_tensor. - REQUIRED methods and attributes - forward, d_model, d_output: controls standard forward pass, - a sequence-to-sequence transformation - __init__ should also satisfy the following interface; - see SequenceIdentity for an example - def __init__(self, d_model, transposed=False, **kwargs) + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model, d_output): + super().__init__() + self.d_model = d_model + self.d_output = d_output + self.linear = nn.Linear(d_model, d_output) - OPTIONAL methods - default_state, step: allows stepping the model recurrently with a hidden state - state_to_tensor, d_state: allows decoding from hidden state + def forward(self, x, state=None): + return self.linear(x), None + + model = MySequenceModel(d_model=64, d_output=32) + x = torch.randn(16, 100, 64) # (batch, sequence, d_model) + output, _ = model(x) # output shape: (16, 100, 32) + + Note: + Subclasses must set self._d_model and self._d_output in their __init__ method. """ @property def d_model(self): - """Model dimension (generally same as input dimension). + """ + Model dimension (generally same as input dimension). + + This property is required for all SequenceModule instantiations. It is used by + the rest of the pipeline (e.g., model backbone, encoder) to track the internal + shapes of the full model. + + Returns: + int: The model dimension. - This attribute is required for all SequenceModule instantiations. - It is used by the rest of the pipeline - (e.g. model backbone, encoder) to track the internal shapes of the full model. + Raises: + NotImplementedError: If the SequenceModule instantiation has not set d_model. + + Note: + Subclasses must set self._d_model in their __init__ method. + + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model): + super().__init__() + self._d_model = d_model # Set the internal attribute + + # ... other methods ... + + model = MySequenceModel(64) + print(model.d_model) # Output: 64 """ if getattr(self, "_d_model", None) is None: raise NotImplementedError("SequenceModule instantiation must set d_model") @@ -39,15 +91,65 @@ def d_model(self): @d_model.setter def d_model(self, d): + """ + Model dimension (generally same as input dimension). + + This property is required for all SequenceModule instantiations. It is used by + the rest of the pipeline (e.g., model backbone, encoder) to track the internal + shapes of the full model. + + Returns: + int: The model dimension. + + Raises: + NotImplementedError: If the SequenceModule instantiation has not set d_model. + + Note: + Subclasses must set self._d_model in their __init__ method. + + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model): + super().__init__() + self._d_model = d_model # Set the internal attribute + + # ... other methods ... + + model = MySequenceModel(64) + print(model.d_model) # Output: 64 + """ self._d_model = d @property def d_output(self): - """Output dimension of model. + """ + Output dimension of the model. + + This property is required for all SequenceModule instantiations. It is used by + the rest of the pipeline (e.g., model backbone, decoder) to track the internal + shapes of the full model. + + Returns: + int: The output dimension of the model. + + Raises: + NotImplementedError: If the SequenceModule instantiation has not specified + d_output for the decoder. - This attribute is required for all SequenceModule instantiations. - It is used by the rest of the pipeline - (e.g. model backbone, decoder) to track the internal shapes of the full model. + Note: + Subclasses must set self._d_output in their __init__ method. + + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model, d_output): + super().__init__() + self._d_model = d_model + self._d_output = d_output # Set the internal attribute + + # ... other methods ... + + model = MySequenceModel(d_model=64, d_output=32) + print(model.d_output) # Output: 32 """ if getattr(self, "_d_output", None) is None: raise NotImplementedError( @@ -57,58 +159,284 @@ def d_output(self): @d_output.setter def d_output(self, d): - self._d_output = d + """ + Output dimension of the model. - def forward(self, x, state=None, **kwargs): - """Forward pass. + This property is required for all SequenceModule instantiations. It is used by + the rest of the pipeline (e.g., model backbone, decoder) to track the internal + shapes of the full model. + + Returns: + int: The output dimension of the model. + + Raises: + NotImplementedError: If the SequenceModule instantiation has not specified + d_output for the decoder. + + Note: + Subclasses must set self._d_output in their __init__ method. + + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model, d_output): + super().__init__() + self._d_model = d_model + self._d_output = d_output # Set the internal attribute - A sequence-to-sequence transformation with an optional state. + # ... other methods ... - Generally, this should map a tensor of shape - (batch, length, self.d_model) to (batch, length, self.d_output) + model = MySequenceModel(d_model=64, d_output=32) + print(model.d_output) # Output: 32 + """ + self._d_output = d - Additionally, it returns a "state" which can be any additional information - For example, RNN and SSM layers may return their hidden state, - while some types of transformer layers - (e.g. Transformer-XL) may want to pass a state as well + def forward(self, x, state=None, **kwargs): + """ + Perform the forward pass of the sequence model. + + This method implements a sequence-to-sequence transformation with an optional state. + It should map a tensor of shape (batch, length, self.d_model) to + (batch, length, self.d_output). + + Args: + x (torch.Tensor): Input tensor of shape (batch, length, self.d_model). + state (Any, optional): Initial state for the forward pass. Defaults to None. + **kwargs: Additional keyword arguments. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor of shape (batch, length, self.d_output). + - Any: Updated state. This can be any additional information, such as + the hidden state for RNN and SSM layers, or state information for + certain types of transformer layers (e.g., Transformer-XL). + + Example: + class MySequenceModel(SequenceModule): + def __init__(self, d_model, d_output): + super().__init__() + self.d_model = d_model + self.d_output = d_output + self.linear = nn.Linear(d_model, d_output) + + def forward(self, x, state=None): + output = self.linear(x) + return output, None + + model = MySequenceModel(d_model=64, d_output=32) + x = torch.randn(16, 100, 64) # (batch, length, d_model) + output, new_state = model(x) + print(output.shape) # torch.Size([16, 100, 32]) """ return x, None @property def state_to_tensor(self): - """Return a function mapping a state to a single tensor. - - This method should be implemented if one wants to use - the hidden state insteadof the output sequence for final prediction. - Currently only used with the StateDecoder. + """ + Function to map a state to a single tensor. + + This property should return a function that converts the model's state into a + single tensor representation. It is primarily used when the hidden state, rather + than the output sequence, is needed for final prediction. This method is + currently only used in conjunction with the StateDecoder. + + Returns: + Callable: A function that takes a state as input and returns a tensor + representation of that state. If not implemented, returns a lambda + function that always returns None. + + Note: + Subclasses should override this property if they want to provide a custom + state-to-tensor conversion. + + Example: + class MyRNNModel(SequenceModule): + def __init__(self, d_model, d_hidden): + super().__init__() + self.rnn = nn.RNN(d_model, d_hidden) + self._d_state = d_hidden + + @property + def state_to_tensor(self): + return lambda state: state[0].squeeze(0) + + # ... other methods ... + + model = MyRNNModel(d_model=64, d_hidden=32) + x = torch.randn(16, 100, 64) # (batch, length, d_model) + _, final_state = model(x) + state_tensor = model.state_to_tensor(final_state) + print(state_tensor.shape) # torch.Size([16, 32]) """ return lambda _: None @property def d_state(self): - """Return dimension of output of self.state_to_tensor.""" + """ + Dimension of the output of self.state_to_tensor. + + This property returns the dimension of the tensor produced by the state_to_tensor + method. It is useful for determining the size of the state representation, + particularly when using the state for downstream tasks or in the StateDecoder. + + Returns: + int or None: The dimension of the state tensor. Returns None if not implemented + or if the state does not have a fixed dimension. + + Note: + Subclasses should override this property if they implement a custom + state_to_tensor method with a known output dimension. + + Example: + class MyLSTMModel(SequenceModule): + def __init__(self, d_model, d_hidden): + super().__init__() + self.lstm = nn.LSTM(d_model, d_hidden) + self._d_state = d_hidden + + @property + def d_state(self): + return self._d_state + + @property + def state_to_tensor(self): + return lambda state: state[0].view(state[0].size(1), -1) + + # ... other methods ... + + model = MyLSTMModel(d_model=64, d_hidden=32) + print(model.d_state) # Output: 32 + """ return None def default_state(self, *batch_shape, device=None): - """Create initial state for a batch of inputs.""" + """ + Create initial state for a batch of inputs. + + This method generates the default initial state for the sequence model. It is + particularly useful for models that maintain internal states, such as RNNs or + state space models. + + Args: + *batch_shape: Variable length argument list for the batch dimensions. + device (torch.device, optional): The device on which to create the state. + Defaults to None, which means the state will be created on the default device. + + Returns: + Any: The initial state for the model. The exact type and shape depend on the + specific implementation. Returns None by default if not implemented. + + Example: + class MyGRUModel(SequenceModule): + def __init__(self, d_model, d_hidden): + super().__init__() + self.gru = nn.GRU(d_model, d_hidden) + self.d_hidden = d_hidden + + def default_state(self, *batch_shape, device=None): + return torch.zeros(*batch_shape, self.d_hidden, device=device) + + # ... other methods ... + + model = MyGRUModel(d_model=64, d_hidden=32) + initial_state = model.default_state(16, device=torch.device('cuda')) + print(initial_state.shape) # torch.Size([16, 32]) + + Note: + Subclasses should override this method if they require a non-None initial state. + The method should be able to handle variable batch dimensions. + """ return None def step(self, x, state=None, **kwargs): - """Step the model recurrently for one step of the input sequence. - - For example, this should correspond to unrolling an RNN for one step. - If the forward pass has signature (B, L, H1) -> (B, L, H2), - this method should generally have signature - (B, H1) -> (B, H2) with an optional recurrent state. + """ + Process one step of the input sequence. + + This method steps the model recurrently for a single step of the input sequence. + It is particularly useful for models that need to process sequences step-by-step, + such as in autoregressive generation or online inference scenarios. + + Args: + x (torch.Tensor): Input tensor for a single step. If the forward pass has + signature (B, L, H1) -> (B, L, H2), this method's input should generally + have shape (B, H1), where B is the batch size and H1 is the input dimension. + state (Any, optional): The current state of the model. Defaults to None. + **kwargs: Additional keyword arguments. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor for the single step, typically of shape (B, H2), + where H2 is the output dimension. + - Any: The updated state after processing this step. + + Raises: + NotImplementedError: If the subclass does not implement this method. + + Example: + class MyRNNModel(SequenceModule): + def __init__(self, d_model, d_hidden): + super().__init__() + self.rnn_cell = nn.RNNCell(d_model, d_hidden) + self.d_model = d_model + self.d_output = d_hidden + + def step(self, x, state=None): + if state is None: + state = torch.zeros(x.size(0), self.d_output, device=x.device) + output = self.rnn_cell(x, state) + return output, output + + # ... other methods ... + + model = MyRNNModel(d_model=64, d_hidden=32) + x = torch.randn(16, 64) # (batch_size, d_model) + output, new_state = model.step(x) + print(output.shape) # torch.Size([16, 32]) + + Note: + This method should be implemented by subclasses that support step-by-step + processing. The exact signature may vary depending on the specific model architecture. """ raise NotImplementedError def TransposedModule(module): - """Transpose module. - - Wrap a SequenceModule class to accept transposed parameter, - handle state, absorb kwargs + """ + Decorator to transpose input and output of a SequenceModule. + + This decorator wraps a SequenceModule class to handle transposed input and output, + manage state, and absorb additional keyword arguments. It allows the module to + operate on either (B, L, H) or (B, H, L) shaped inputs, where B is batch size, + L is sequence length, and H is hidden dimension. + + Attributes: + transposed (bool): If True, the input and output tensors are transposed. + + Args: + module (type): The SequenceModule class to be wrapped. + + Returns: + type: A new class that wraps the input module with transposition functionality. + + Example: + @TransposedModule + class MySequenceModule(SequenceModule): + def __init__(self, d_model): + super().__init__() + self.d_model = d_model + self.d_output = d_model + + def forward(self, x, state=None): + # Process x + return x, state + + # Now MySequenceModule can handle both (B, L, H) and (B, H, L) inputs + model = MySequenceModule(d_model=64, transposed=True) + x = torch.randn(32, 64, 100) # (B, H, L) + output, _ = model(x) # output will be (32, 64, 100) + + Note: + The wrapped module's forward method should accept 'state' as an argument + and return both the processed tensor and the next state. """ # https://stackoverflow.com/a/65470430/1980685 @@ -134,7 +462,42 @@ def forward(self, x, state=None, **kwargs): @TransposedModule class SequenceIdentity(SequenceModule): - """Simple SequenceModule for testing purposes.""" + """ + A simple identity SequenceModule for testing purposes. + + This class implements a basic SequenceModule that passes the input through + unchanged. It is primarily used for testing and as a minimal example of + a SequenceModule implementation. + + The SequenceIdentity class is wrapped with the @TransposedModule decorator, + allowing it to handle both standard and transposed input formats. + + Attributes: + d_model (int): The input and output dimension of the model. + d_output (int): Alias for d_model, as this module doesn't change dimensions. + + Args: + d_model (int): The dimension of the input and output. + dropout (float, optional): Dropout rate (unused in this implementation). + Defaults to 0.0. + **kwargs: Additional keyword arguments (unused). + + Example: + model = SequenceIdentity(d_model=64) + x = torch.randn(32, 100, 64) # (batch_size, sequence_length, d_model) + output, _ = model(x) + assert torch.allclose(x, output) # True, as this is an identity module + + # With transposed input + model_transposed = SequenceIdentity(d_model=64, transposed=True) + x_transposed = torch.randn(32, 64, 100) # (batch_size, d_model, sequence_length) + output_transposed, _ = model_transposed(x_transposed) + assert torch.allclose(x_transposed, output_transposed) # True + + Note: + This class is wrapped with @TransposedModule, which allows it to handle + both (B, L, H) and (B, H, L) input formats based on the 'transposed' parameter. + """ def __init__(self, d_model, dropout=0.0, **kwargs): """Initialize SequenceModule. @@ -147,13 +510,83 @@ def __init__(self, d_model, dropout=0.0, **kwargs): self.d_output = d_model def forward(self, x, state=None): - """Forward pass.""" + """ + Perform the forward pass of the SequenceIdentity module. + + This method implements the identity operation, returning the input unchanged. + + Args: + x (torch.Tensor): Input tensor of shape (batch, length, d_model) if not transposed, + or (batch, d_model, length) if transposed. + state (Any, optional): Unused in this implementation. Defaults to None. + + Returns: + tuple: A tuple containing: + - torch.Tensor: The input tensor x, unchanged. + - None: As this module doesn't maintain state. + + Example: + model = SequenceIdentity(d_model=64) + x = torch.randn(32, 100, 64) # (batch_size, sequence_length, d_model) + output, _ = model(x) + assert torch.allclose(x, output) # True, as this is an identity operation + + Note: + If the module was initialized with transposed=True, the method expects + and returns tensors in the shape (batch, d_model, length). + """ return x, state def default_state(self, *batch_shape, device=None): - """Create initial state for a batch of inputs.""" + """ + Create initial state for a batch of inputs. + + This method returns None as the SequenceIdentity module does not maintain any state. + + Args: + *batch_shape: Variable length argument list for the batch dimensions (unused). + device (torch.device, optional): The device on which to create the state (unused). + Defaults to None. + + Returns: + None: As SequenceIdentity doesn't use any state. + + Example: + model = SequenceIdentity(d_model=64) + initial_state = model.default_state(32) # batch size of 32 + assert initial_state is None # True + + Note: + This method is implemented to conform to the SequenceModule interface, + but it doesn't perform any meaningful operation for SequenceIdentity. + """ return None def step(self, x, state=None, **kwargs): - """Step the model recurrently for one step of the input sequence.""" + """ + Process one step of the input sequence. + + This method implements the identity operation for a single step, returning the input unchanged. + + Args: + x (torch.Tensor): Input tensor for a single step, typically of shape (batch, d_model). + state (Any, optional): Unused in this implementation. Defaults to None. + **kwargs: Additional keyword arguments (unused). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The input tensor x, unchanged. + - None: As this module doesn't maintain state. + + Example: + model = SequenceIdentity(d_model=64) + x = torch.randn(32, 64) # (batch_size, d_model) + output, _ = model.step(x) + assert torch.allclose(x, output) # True, as this is an identity operation + + Note: + This method is implemented to conform to the SequenceModule interface. + For SequenceIdentity, it performs the same operation as the forward method, + but for a single step of the sequence. + """ return x, state diff --git a/espnet2/asr/state_spaces/block.py b/espnet2/asr/state_spaces/block.py index 5f64aadbdad..cf25155c17d 100644 --- a/espnet2/asr/state_spaces/block.py +++ b/espnet2/asr/state_spaces/block.py @@ -26,22 +26,47 @@ class SequenceResidualBlock(SequenceModule): - """Residual block wrapper for black box layer. - - The SequenceResidualBlock class implements a generic - (batch, length, d_input) -> (batch, length, d_input) transformation + """ + Residual block wrapper for black box layer. + + This class implements a generic (batch, length, d_input) -> (batch, length, d_input) transformation + with configurable normalization, pooling, and residual options. + + Attributes: + i_layer (int): Layer index, used by certain residual functions like Decay. + d_input (int): Input feature dimension. + layer (SequenceModule): Black box layer module. + prenorm (bool): If True, apply normalization before the layer; otherwise, after. + transposed (bool): If True, transpose inputs so each layer receives (batch, dim, length). + residual (ResidualFunction): Residual function module. + norm (Normalization): Normalization layer. + pool (PoolModule): Pooling layer. + drop (nn.Module): Dropout module. + drop_path (nn.Module): Stochastic depth module. Args: - d_input: Input feature dimension - i_layer: Layer index, only needs to be passed into certain residuals like Decay - dropout: Dropout for black box module - tie_dropout: Tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d - transposed: Transpose inputs so each layer receives (batch, dim, length) - layer: Config for black box module - residual: Config for residual function - norm: Config for normalization layer - pool: Config for pooling layer per stage - drop_path: Drop ratio for stochastic depth + d_input (int): Input feature dimension. + i_layer (int, optional): Layer index. Defaults to None. + prenorm (bool, optional): Whether to apply normalization before the layer. Defaults to True. + dropout (float, optional): Dropout rate. Defaults to 0.0. + tie_dropout (bool, optional): Whether to tie dropout mask across sequence. Defaults to False. + transposed (bool, optional): Whether to transpose inputs. Defaults to False. + layer (dict, optional): Config for black box module. Defaults to None. + residual (dict, optional): Config for residual function. Defaults to None. + norm (str or dict, optional): Config for normalization layer. Defaults to None. + pool (dict, optional): Config for pooling layer. Defaults to None. + drop_path (float, optional): Drop ratio for stochastic depth. Defaults to 0.0. + + Example: + >>> block = SequenceResidualBlock(d_input=256, prenorm=True, dropout=0.1) + >>> x = torch.randn(32, 100, 256) # (batch, length, d_input) + >>> y, _ = block(x) + >>> y.shape + torch.Size([32, 100, 256]) + + Note: + The class supports various configurations for normalization, residual connections, + pooling, and regularization, making it highly flexible for different architectural designs. """ def __init__( @@ -111,20 +136,94 @@ def __init__( @property def d_output(self): + """ + Output dimension of the SequenceResidualBlock. + + Returns: + int: The output dimension. If a pooling layer is present, it returns the output + dimension of the pooling layer. Otherwise, it returns the residual dimension. + + Note: + This property dynamically computes the output dimension based on the block's + configuration, taking into account whether pooling is applied. + """ return self.pool.d_output if self.pool is not None else self.d_residual @property def d_state(self): + """ + State dimension of the underlying layer. + + Returns: + int: The state dimension of the black box layer. + + Note: + This property provides access to the state dimension of the internal layer, + which is useful for understanding the hidden state size in stateful models. + """ return self.layer.d_state @property def state_to_tensor(self): + """ + Method to convert the layer's state to a tensor. + + Returns: + callable: The state_to_tensor method of the underlying layer. + + Note: + This property provides access to the state_to_tensor method of the internal layer, + allowing for consistent state handling across the residual block wrapper. + """ return self.layer.state_to_tensor def default_state(self, *args, **kwargs): + """ + Get the default state for the underlying layer. + + This method delegates to the default_state method of the internal layer. + + Args: + *args: Variable length argument list to be passed to the layer's default_state method. + **kwargs: Arbitrary keyword arguments to be passed to the layer's default_state method. + + Returns: + The default state of the underlying layer. + + Note: + This method allows the SequenceResidualBlock to maintain the same interface + for state initialization as its internal layer. + """ return self.layer.default_state(*args, **kwargs) def forward(self, x, state=None, **kwargs): + """ + Forward pass of the SequenceResidualBlock. + + This method applies the full sequence of operations: normalization (if prenorm), + black box layer, residual connection, normalization (if postnorm), and pooling. + + Args: + x (torch.Tensor): Input tensor of shape (batch, length, d_input). + state (Any, optional): Initial state for the layer. Defaults to None. + **kwargs: Additional keyword arguments to be passed to the black box layer. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor after all operations. + - Any: Updated state of the black box layer. + + Example: + >>> block = SequenceResidualBlock(d_input=256) + >>> x = torch.randn(32, 100, 256) + >>> output, new_state = block(x) + >>> output.shape + torch.Size([32, 100, 256]) + + Note: + The order of operations and the application of each component (norm, residual, pool) + depends on the block's configuration. + """ y = x # Pre-norm @@ -149,6 +248,35 @@ def forward(self, x, state=None, **kwargs): return y, state def step(self, x, state, **kwargs): + """ + Perform a single step forward pass of the SequenceResidualBlock. + + This method is designed for sequential processing, applying the block's operations + on a single time step input. + + Args: + x (torch.Tensor): Input tensor for a single time step. + state (Any): Current state of the layer. + **kwargs: Additional keyword arguments to be passed to the black box layer's step method. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor after all operations for the current time step. + - Any: Updated state of the black box layer. + + Note: + This method follows a similar sequence of operations as the forward method, + but is adapted for step-by-step processing. It does not apply dropout or stochastic depth, + and the residual connection is applied without transposition. + + Example: + >>> block = SequenceResidualBlock(d_input=256) + >>> x = torch.randn(32, 256) # Single time step input + >>> state = block.default_state(batch_size=32) + >>> output, new_state = block.step(x, state) + >>> output.shape + torch.Size([32, 256]) + """ y = x # Pre-norm diff --git a/espnet2/asr/state_spaces/cauchy.py b/espnet2/asr/state_spaces/cauchy.py index 4d36ea92657..23a29f33f12 100644 --- a/espnet2/asr/state_spaces/cauchy.py +++ b/espnet2/asr/state_spaces/cauchy.py @@ -13,13 +13,39 @@ def cauchy_mult_torch( v: torch.Tensor, z: torch.Tensor, w: torch.Tensor, symmetric=True ) -> torch.Tensor: - """Compute Cauchy kernel. + """ + Compute the Cauchy kernel using PyTorch operations. + + This function calculates the Cauchy kernel for given input tensors. It supports + both symmetric and non-symmetric computations. + + Args: + v (torch.Tensor): Input tensor of shape (B, N), where B is the batch size + and N is the number of elements. + z (torch.Tensor): Input tensor of shape (L,), where L is the length. + w (torch.Tensor): Input tensor of shape (B, N), where B is the batch size + and N is the number of elements. + symmetric (bool, optional): If True, assumes v and w contain complex conjugate + pairs of the form [v_half, v_half.conj()] and [w_half, w_half.conj()]. + Defaults to True. + + Returns: + torch.Tensor: The computed Cauchy kernel. - v: (B, N) - z: (L) - w: (B, N) - symmetric: whether to assume that v and w contain complex conjugate pairs, of the - form [v_half, v_half.conj()] and [w_half, w_half.conj()] + Raises: + AssertionError: If symmetric is True and N is not even. + + Examples: + >>> v = torch.randn(2, 4) + >>> z = torch.randn(3) + >>> w = torch.randn(2, 4) + >>> result = cauchy_mult_torch(v, z, w) + >>> print(result.shape) + torch.Size([2, 3]) + + Note: + When symmetric is True, the function only uses the first half of v and w, + assuming they contain complex conjugate pairs. """ if not symmetric: return ( @@ -39,6 +65,34 @@ def cauchy_mult_torch( def cauchy_mult_keops(v, z, w): + """ + Compute the Cauchy kernel using KeOps LazyTensors. + + This function calculates the Cauchy kernel for given input tensors using + KeOps LazyTensors for efficient GPU computation. + + Args: + v (torch.Tensor): Input tensor of shape (b, N), where b is the batch size + and N is the number of elements. + z (torch.Tensor): Input tensor of shape (L,), where L is the length. + w (torch.Tensor): Input tensor of shape (b, N), where b is the batch size + and N is the number of elements. + + Returns: + torch.Tensor: The computed Cauchy kernel of shape (b, L). + + Note: + This function requires the PyKeOps library to be installed and + configured properly for GPU acceleration. + + Examples: + >>> v = torch.randn(2, 1000) + >>> z = torch.randn(500) + >>> w = torch.randn(2, 1000) + >>> result = cauchy_mult_keops(v, z, w) + >>> print(result.shape) + torch.Size([2, 500]) + """ from pykeops.torch import LazyTensor v_l = LazyTensor(rearrange(v, "b N -> b 1 N 1")) @@ -58,7 +112,38 @@ def _cauchy_mult(v, z, w, symmetric=True): def cauchy_mult(v, z, w, symmetric=True): - """Wrap the cuda method to deal with shapes.""" + """ + Wrapper function for Cauchy multiplication that handles tensor shapes. + + This function wraps the CUDA-based Cauchy multiplication method, ensuring proper + tensor shapes and broadcasting before computation. + + Args: + v (torch.Tensor): Input tensor of shape (..., N), where N is the number of elements. + z (torch.Tensor): Input tensor of shape (L,), where L is the length. + w (torch.Tensor): Input tensor of shape (..., N), where N is the number of elements. + symmetric (bool, optional): If True, uses symmetric Cauchy multiplication. + Defaults to True. + + Returns: + torch.Tensor: The result of Cauchy multiplication with shape (..., L). + + Raises: + AssertionError: If z is not a 1-dimensional tensor after squeezing. + AssertionError: If the last dimensions of v and w do not match. + + Note: + This function broadcasts v and w tensors, making it flexible for various + input shapes. + + Examples: + >>> v = torch.randn(2, 3, 1000) + >>> z = torch.randn(500) + >>> w = torch.randn(2, 3, 1000) + >>> result = cauchy_mult(v, z, w) + >>> print(result.shape) + torch.Size([2, 3, 500]) + """ v, w = torch.broadcast_tensors(v, w) shape = v.shape # z_shape = z.shape @@ -78,8 +163,52 @@ def cauchy_mult(v, z, w, symmetric=True): class CauchyMultiply(torch.autograd.Function): + """ + Custom autograd function for Cauchy multiplication. + + This class implements a custom autograd function for Cauchy multiplication, + providing both forward and backward passes. It utilizes CUDA operations + for efficient computation on GPU. + + Note: + This class is designed to be used with PyTorch's autograd system and + should not be instantiated directly. Instead, use it through the + `_cauchy_mult` function. + + Attributes: + Inherits attributes from torch.autograd.Function. + + Raises: + NotImplementedError: If input tensors are not CUDA tensors, or if the + input dimensions are not supported. + """ + @staticmethod def forward(ctx, v, z, w): + """ + Perform the forward pass of Cauchy multiplication. + + This method computes the Cauchy multiplication using CUDA operations. + + Args: + ctx (torch.autograd.function.FunctionCtx): Context object to save + information for backward computation. + v (torch.Tensor): Input tensor of shape (batch, N). + z (torch.Tensor): Input tensor of shape (L,). + w (torch.Tensor): Input tensor of shape (batch, N). + + Returns: + torch.Tensor: Result of Cauchy multiplication. + + Raises: + NotImplementedError: If N is not in the supported values list. + NotImplementedError: If L is not a multiple of 32. + NotImplementedError: If input tensors are not CUDA tensors. + + Note: + Currently only supports N values of 64 (2^6) and L values that are + multiples of 32. + """ batch, N = v.shape # supported_N_values = [1 << log_n for log_n in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] supported_N_values = [1 << log_n for log_n in [6]] @@ -95,14 +224,79 @@ def forward(ctx, v, z, w): @staticmethod def backward(ctx, dout): + """ + Perform the backward pass of Cauchy multiplication. + + This method computes the gradients with respect to inputs v and w. + + Args: + ctx (torch.autograd.function.FunctionCtx): Context object containing + saved tensors from the forward pass. + dout (torch.Tensor): Gradient of the loss with respect to the output + of the forward pass. + + Returns: + tuple: A tuple containing: + - dv (torch.Tensor): Gradient with respect to input v. + - None: Placeholder for gradient with respect to z (not computed). + - dw (torch.Tensor): Gradient with respect to input w. + + Note: + The gradient with respect to z is not computed and returned as None. + """ v, z, w = ctx.saved_tensors dv, dw = cauchy_mult_bwd(v, z, w, dout) return dv, None, dw class CauchyMultiplySymmetric(torch.autograd.Function): + """ + Custom autograd function for symmetric Cauchy multiplication. + + This class implements a custom autograd function for symmetric Cauchy + multiplication, providing both forward and backward passes. It utilizes + CUDA operations for efficient computation on GPU, assuming symmetry in + the input tensors. + + Note: + This class is designed to be used with PyTorch's autograd system and + should not be instantiated directly. Instead, use it through the + `_cauchy_mult` function with the symmetric option set to True. + + Attributes: + Inherits attributes from torch.autograd.Function. + + Raises: + NotImplementedError: If input tensors are not CUDA tensors, or if the + input dimensions are not supported. + """ + @staticmethod def forward(ctx, v, z, w): + """ + Perform the forward pass of symmetric Cauchy multiplication. + + This method computes the symmetric Cauchy multiplication using CUDA operations. + + Args: + ctx (torch.autograd.function.FunctionCtx): Context object to save + information for backward computation. + v (torch.Tensor): Input tensor of shape (batch, N). + z (torch.Tensor): Input tensor of shape (L,). + w (torch.Tensor): Input tensor of shape (batch, N). + + Returns: + torch.Tensor: Result of symmetric Cauchy multiplication. + + Raises: + NotImplementedError: If N is not in the supported values list. + NotImplementedError: If L exceeds the maximum supported value. + NotImplementedError: If input tensors are not CUDA tensors. + + Note: + Supports N values that are powers of 2 from 2 to 1024. + The maximum supported L value is 32 * 1024 * 64 * 1024. + """ batch, N = v.shape supported_N_values = [1 << log_n for log_n in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] L = z.shape[-1] @@ -118,6 +312,29 @@ def forward(ctx, v, z, w): @staticmethod def backward(ctx, dout): + """ + Perform the backward pass of symmetric Cauchy multiplication. + + This method computes the gradients with respect to inputs v and w for the + symmetric case. + + Args: + ctx (torch.autograd.function.FunctionCtx): Context object containing + saved tensors from the forward pass. + dout (torch.Tensor): Gradient of the loss with respect to the output + of the forward pass. + + Returns: + tuple: A tuple containing: + - dv (torch.Tensor): Gradient with respect to input v. + - None: Placeholder for gradient with respect to z (not computed). + - dw (torch.Tensor): Gradient with respect to input w. + + Note: + The gradient with respect to z is not computed and returned as None. + This method assumes symmetry in the input tensors and uses specialized + CUDA operations for symmetric Cauchy multiplication. + """ v, z, w = ctx.saved_tensors dv, dw = cauchy_mult_sym_bwd(v, z, w, dout) return dv, None, dw diff --git a/espnet2/asr/state_spaces/components.py b/espnet2/asr/state_spaces/components.py index b6062be2b18..f20236fc688 100644 --- a/espnet2/asr/state_spaces/components.py +++ b/espnet2/asr/state_spaces/components.py @@ -11,23 +11,35 @@ def stochastic_depth(input: torch.tensor, p: float, mode: str, training: bool = True): - """Apply stochastic depth. + """ + Apply stochastic depth to the input tensor. - Implements the Stochastic Depth from `"Deep Networks with Stochastic Depth" - `_ used for randomly dropping residual - branches of residual architectures. + Implements the Stochastic Depth technique from "Deep Networks with Stochastic Depth" + (https://arxiv.org/abs/1603.09382) used for randomly dropping residual branches + of residual architectures. Args: - input (Tensor[N, ...]): The input tensor or arbitrary dimensions with the first - one being its batch i.e. a batch with ``N`` rows. - p (float): probability of the input to be zeroed. - mode (str): ``"batch"`` or ``"row"``. - ``"batch"`` randomly zeroes the entire input, ``"row"`` zeroes - randomly selected rows from the batch. - training: apply stochastic depth if is ``True``. Default: ``True`` + input (torch.tensor): The input tensor of arbitrary dimensions with the first + dimension being its batch size. + p (float): Probability of the input to be zeroed. + mode (str): Either "batch" or "row". "batch" randomly zeroes the entire input, + "row" zeroes randomly selected rows from the batch. + training (bool, optional): Apply stochastic depth if True. Defaults to True. Returns: - Tensor[N, ...]: The randomly zeroed tensor. + torch.tensor: The randomly zeroed tensor. + + Raises: + ValueError: If p is not between 0 and 1, or if mode is not "batch" or "row". + + Examples: + >>> input_tensor = torch.randn(32, 64, 224, 224) + >>> output = stochastic_depth(input_tensor, p=0.2, mode="batch") + >>> print(output.shape) + torch.Size([32, 64, 224, 224]) + + Note: + When training is False or p is 0, the function returns the input tensor unchanged. """ if p < 0.0 or p > 1.0: raise ValueError( @@ -51,9 +63,21 @@ def stochastic_depth(input: torch.tensor, p: float, mode: str, training: bool = class StochasticDepth(nn.Module): - """Stochastic depth module. + """ + A module that applies stochastic depth to its input. - See :func:`stochastic_depth`. + This module implements the Stochastic Depth technique from the paper + "Deep Networks with Stochastic Depth" (https://arxiv.org/abs/1603.09382). + It randomly drops entire layers or parts of the input during training, + which can help in regularizing deep networks. + + Attributes: + p (float): The probability of dropping the input. + mode (str): The mode of dropping, either "batch" or "row". + + Note: + This implementation is a custom version and may need to be upgraded + to use the official `torchvision.ops.StochasticDepth` in future versions. """ def __init__(self, p: float, mode: str) -> None: @@ -64,6 +88,22 @@ def __init__(self, p: float, mode: str) -> None: self.mode = mode def forward(self, input): + """ + Apply stochastic depth to the input tensor. + + This method applies the stochastic depth technique to the input tensor + based on the probability and mode specified during initialization. + + Args: + input (torch.Tensor): The input tensor to apply stochastic depth to. + + Returns: + torch.Tensor: The output tensor after applying stochastic depth. + + Note: + The behavior of this method depends on whether the module is in + training mode or not, as well as the specified probability and mode. + """ return stochastic_depth(input, self.p, self.mode, self.training) def __repr__(self) -> str: @@ -75,6 +115,23 @@ def __repr__(self) -> str: class DropoutNd(nn.Module): + """ + A module that applies n-dimensional dropout to its input. + + This module extends the concept of dropout to n-dimensional tensors, + allowing for more flexible dropout patterns in complex neural network + architectures. + + Attributes: + p (float): The probability of an element to be zeroed. Must be between 0 and 1. + tie (bool): If True, ties the dropout mask across sequence lengths. + transposed (bool): If True, assumes the input tensor is in a transposed format. + + Note: + This implementation uses a custom dropout mechanism and may behave + differently from PyTorch's built-in dropout functions. + """ + def __init__(self, p: float = 0.5, tie=True, transposed=True): """Initialize dropout module. @@ -91,9 +148,28 @@ def __init__(self, p: float = 0.5, tie=True, transposed=True): self.binomial = torch.distributions.binomial.Binomial(probs=1 - self.p) def forward(self, X): - """Forward pass. + """ + Apply n-dimensional dropout to the input tensor. + + This method applies dropout to the input tensor based on the probability + and configuration specified during initialization. - X: (batch, dim, lengths...) + Args: + X (torch.Tensor): The input tensor of shape (batch, dim, lengths...). + + Returns: + torch.Tensor: The output tensor after applying dropout. + + Note: + The dropout is only applied during training mode. In evaluation mode, + the input tensor is returned unchanged. + + Examples: + >>> dropout = DropoutNd(p=0.5, tie=True, transposed=False) + >>> input = torch.randn(32, 64, 10, 10) + >>> output = dropout(input) + >>> print(output.shape) + torch.Size([32, 64, 10, 10]) """ if self.training: if not self.transposed: @@ -111,6 +187,42 @@ def forward(self, X): def Activation(activation=None, size=None, dim=-1): + """ + Create and return an activation layer based on the specified activation type. + + This function provides a convenient way to create various activation layers + commonly used in neural networks. + + Args: + activation (str, optional): The type of activation to use. Supported values are: + None, "id", "identity", "linear", "tanh", "relu", "gelu", "swish", "silu", + "glu", "sigmoid", "sqrelu", "ln". Defaults to None. + size (int, optional): The size parameter for certain activation types. + Only used for specific activations. Defaults to None. + dim (int, optional): The dimension along which to apply the activation for + dimension-specific activations like GLU. Defaults to -1. + + Returns: + nn.Module: An instance of the specified activation layer. + + Raises: + NotImplementedError: If the specified activation is not implemented. + + Examples: + >>> relu_activation = Activation("relu") + >>> output = relu_activation(torch.randn(10, 20)) + >>> print(type(output)) + + + >>> glu_activation = Activation("glu", dim=1) + >>> output = glu_activation(torch.randn(10, 20, 30)) + >>> print(output.shape) + torch.Size([10, 10, 30]) + + Note: + Some activation types (e.g., "ln") may require additional parameters + that are not directly exposed in this function's interface. + """ if activation in [None, "id", "identity", "linear"]: return nn.Identity() elif activation == "tanh": @@ -136,6 +248,39 @@ def Activation(activation=None, size=None, dim=-1): def get_initializer(name, activation=None): + """ + Get an initialization function based on the specified name and activation. + + This function returns an initializer function that can be used to initialize + the weights of neural network layers. + + Args: + name (str): The name of the initializer. Supported values are: + "uniform", "normal", "xavier", "zero", "one". + activation (str, optional): The activation function used in the layer. + This is used to determine the appropriate gain for certain initializers. + Supported values are: None, "id", "identity", "linear", "modrelu", "relu", + "tanh", "sigmoid", "gelu", "swish", "silu". Defaults to None. + + Returns: + callable: An initialization function that can be applied to tensor weights. + + Raises: + NotImplementedError: If the specified name or activation is not supported. + + Examples: + >>> init_func = get_initializer("uniform", activation="relu") + >>> linear = nn.Linear(10, 20) + >>> init_func(linear.weight) + + >>> xavier_init = get_initializer("xavier") + >>> conv = nn.Conv2d(3, 64, 3) + >>> xavier_init(conv.weight) + + Note: + The returned initializer functions are partial functions from PyTorch's + initialization methods, with pre-set parameters based on the input arguments. + """ if activation in [None, "id", "identity", "linear", "modrelu"]: nonlinearity = "linear" elif activation in ["relu", "tanh", "sigmoid"]: @@ -177,7 +322,44 @@ def LinearActivation( weight_norm=False, **kwargs, ): - """Return a linear module, initialization, and activation.""" + """ + Create a linear module with optional initialization and activation. + + This function constructs a linear layer with various options for initialization, + activation, and normalization. + + Args: + d_input (int): Dimension of the input features. + d_output (int): Dimension of the output features. + bias (bool, optional): If True, adds a learnable bias to the layer. Defaults to True. + zero_bias_init (bool, optional): If True, initializes the bias to zero. Defaults to False. + transposed (bool, optional): If True, uses TransposedLinear instead of nn.Linear. Defaults to False. + initializer (str, optional): Initialization method for the weights. Defaults to None. + activation (str, optional): Activation function to use. Defaults to None. + activate (bool, optional): If True, applies the activation as part of this module. Defaults to False. + weight_norm (bool, optional): If True, applies weight normalization. Defaults to False. + **kwargs: Additional keyword arguments passed to the linear layer. + + Returns: + nn.Module: A linear module, possibly followed by an activation function. + + Examples: + >>> layer = LinearActivation(100, 200, activation='relu', activate=True) + >>> input = torch.randn(32, 100) + >>> output = layer(input) + >>> print(output.shape) + torch.Size([32, 200]) + + >>> transposed_layer = LinearActivation(50, 75, transposed=True, weight_norm=True) + >>> input = torch.randn(16, 50, 10) + >>> output = transposed_layer(input) + >>> print(output.shape) + torch.Size([16, 75, 10]) + + Note: + If activation is set to 'glu', the output dimension is doubled internally + before applying the activation. + """ # Construct core module # linear_cls = partial(nn.Conv1d, kernel_size=1) if transposed else nn.Linear linear_cls = TransposedLinear if transposed else nn.Linear @@ -204,15 +386,65 @@ def LinearActivation( class SquaredReLU(nn.Module): + """ + A module that applies a squared ReLU activation function. + + This activation function first applies a standard ReLU (Rectified Linear Unit) + activation, then squares the result. It can be used as a non-linear activation + in neural networks, potentially capturing more complex patterns than standard ReLU. + + The function is defined as: + f(x) = (max(0, x))^2 + + Note: + This activation function may lead to different gradient behavior compared + to standard ReLU, potentially affecting the training dynamics of the network. + """ + def forward(self, x): + """ + Apply the squared ReLU activation function to the input. + + This method applies the ReLU function to the input and then squares the result. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying the squared ReLU activation. + + Examples: + >>> squared_relu = SquaredReLU() + >>> input = torch.randn(10) + >>> output = squared_relu(input) + >>> print(output.shape) + torch.Size([10]) + + Note: + For negative input values, the output will be zero. For positive input values, + the output will be the square of the input. + """ return F.relu(x) ** 2 class TransposedLinear(nn.Module): - """Transposed linear module. - - Linear module on the second-to-last dimension - Assumes shape (B, D, L), where L can be 1 or more axis + """ + A linear module that operates on the second-to-last dimension of the input. + + This module implements a linear transformation similar to nn.Linear, but applies + the transformation on the second-to-last dimension of the input tensor. It is + designed to handle input tensors of shape (B, D, L), where B is the batch size, + D is the input dimension, and L represents one or more additional dimensions. + + Attributes: + weight (nn.Parameter): The learnable weights of the module of shape (d_output, d_input). + bias (nn.Parameter or float): The learnable bias of the module of shape (d_output,). + If bias is False, then the layer does not use a bias. + + Note: + This module can be particularly useful in scenarios where the standard linear + layer's dimension handling is not suitable, such as in certain types of + sequence processing or when working with multi-dimensional data. """ def __init__(self, d_input, d_output, bias=True): @@ -232,6 +464,30 @@ def __init__(self, d_input, d_output, bias=True): self.bias = 0.0 def forward(self, x): + """ + Apply a linear transformation to the input tensor. + + This method performs the linear transformation on the second-to-last dimension + of the input tensor. + + Args: + x (torch.Tensor): The input tensor of shape (B, D, ...), where B is the batch size, + D is the input dimension, and ... represents any number of additional dimensions. + + Returns: + torch.Tensor: The output tensor of shape (B, V, ...), where V is the output dimension. + + Examples: + >>> transposed_linear = TransposedLinear(64, 128) + >>> input = torch.randn(32, 64, 10, 10) + >>> output = transposed_linear(input) + >>> print(output.shape) + torch.Size([32, 128, 10, 10]) + + Note: + The transformation is applied using einsum for efficient computation, + especially for inputs with multiple trailing dimensions. + """ num_axis = len(x.shape[2:]) # num_axis in L, for broadcasting bias y = contract("b u ..., v u -> b v ...", x, self.weight) + self.bias.view( -1, *[1] * num_axis @@ -240,13 +496,23 @@ def forward(self, x): class TransposedLN(nn.Module): - """Transposed LayerNorm module. - - LayerNorm module over second dimension - Assumes shape (B, D, L), where L can be 1 or more axis - - This is slow and a dedicated CUDA/Triton implementation - shuld provide substantial end-to-end speedup + """ + A transposed Layer Normalization module. + + This module applies Layer Normalization over the second dimension of the input tensor. + It is designed to handle input tensors of shape (B, D, L), where B is the batch size, + D is the feature dimension to be normalized, and L represents one or more additional dimensions. + + Attributes: + scalar (bool): If True, uses learnable scalar parameters for affine transformation. + If False, uses a full LayerNorm. + m (nn.Parameter): Learnable shift parameter when scalar is True. + s (nn.Parameter): Learnable scale parameter when scalar is True. + ln (nn.LayerNorm): LayerNorm module used when scalar is False. + + Note: + This implementation may be slower than a dedicated CUDA/Triton implementation. + Future optimizations could provide substantial end-to-end speedup. """ def __init__(self, d, scalar=True): @@ -261,6 +527,30 @@ def __init__(self, d, scalar=True): self.ln = nn.LayerNorm(d) def forward(self, x): + """ + Apply transposed Layer Normalization to the input tensor. + + This method normalizes the input tensor along the second dimension (D). + + Args: + x (torch.Tensor): The input tensor of shape (B, D, ...), where B is the batch size, + D is the feature dimension to be normalized, and ... represents any number of + additional dimensions. + + Returns: + torch.Tensor: The normalized output tensor of the same shape as the input. + + Examples: + >>> transposed_ln = TransposedLN(64, scalar=True) + >>> input = torch.randn(32, 64, 10, 10) + >>> output = transposed_ln(input) + >>> print(output.shape) + torch.Size([32, 64, 10, 10]) + + Note: + When scalar is True, it uses learnable scalar parameters for affine transformation. + When scalar is False, it applies full LayerNorm by rearranging the tensor dimensions. + """ if self.scalar: # calc. stats over D dim / channels s, m = torch.std_mean(x, dim=1, unbiased=False, keepdim=True) @@ -274,6 +564,25 @@ def forward(self, x): class Normalization(nn.Module): + """ + A flexible normalization module supporting various normalization techniques. + + This module provides a unified interface for different types of normalization, + including Layer Normalization, Instance Normalization, Batch Normalization, + and Group Normalization. It can handle both standard and transposed input formats. + + Attributes: + transposed (bool): If True, assumes the length dimension is -1 or -2. + _name_ (str): The type of normalization to use. Options are "layer", "instance", + "batch", "group", or "none". + channel (bool): If True, normalization is applied over the channel dimension. + norm (nn.Module): The actual normalization module based on the specified type. + + Note: + The behavior and performance of this module can vary significantly based on + the chosen normalization type and the structure of the input data. + """ + def __init__( self, d, @@ -313,6 +622,30 @@ def __init__( raise NotImplementedError def forward(self, x): + """ + Apply the specified normalization to the input tensor. + + This method reshapes the input tensor if necessary, applies the chosen + normalization technique, and then restores the original shape. + + Args: + x (torch.Tensor): The input tensor to be normalized. The shape depends on + the 'transposed' attribute and the specific normalization technique. + + Returns: + torch.Tensor: The normalized tensor, maintaining the same shape as the input. + + Examples: + >>> norm = Normalization(64, transposed=False, _name_="layer") + >>> input = torch.randn(32, 10, 64) + >>> output = norm(input) + >>> print(output.shape) + torch.Size([32, 10, 64]) + + Note: + The method handles higher dimensional inputs by reshaping them before + normalization and then restoring the original shape afterwards. + """ # Handle higher dimension logic shape = x.shape if self.transposed: @@ -334,6 +667,36 @@ def forward(self, x): return x def step(self, x, **kwargs): + """ + Apply normalization to a single step input. + + This method is designed for use in scenarios where normalization needs to be + applied to a single time step or element, such as in recurrent neural networks + or sequential processing. + + Args: + x (torch.Tensor): The input tensor representing a single step or element. + **kwargs: Additional keyword arguments that might be required for specific + normalization types. + + Returns: + torch.Tensor: The normalized tensor for the single step. + + Raises: + AssertionError: If the normalization type is not one of "layer", "instance", + "batch", "group", or "none". + + Examples: + >>> norm = Normalization(64, transposed=True, _name_="layer") + >>> step_input = torch.randn(32, 64) + >>> normalized_step = norm.step(step_input) + >>> print(normalized_step.shape) + torch.Size([32, 64]) + + Note: + This method adds an extra dimension to the input if 'transposed' is True, + applies normalization, and then squeezes the added dimension. + """ assert self._name_ in ["layer", "instance", "batch", "group", "none"] if self.transposed: x = x.unsqueeze(-1) @@ -344,6 +707,22 @@ def step(self, x, **kwargs): class TSNormalization(nn.Module): + """ + A module for time series normalization. + + This class implements normalization techniques specifically designed for time series data. + It supports different methods of normalization based on the temporal characteristics of the input. + + Attributes: + method (str): The normalization method to use. Supported values are "mean" and "last". + horizon (int): The number of time steps to consider for normalization. + + Note: + This normalization is particularly useful in time series forecasting tasks, + where different parts of the time series may need to be treated differently + for effective normalization. + """ + def __init__(self, method, horizon): super().__init__() @@ -351,6 +730,30 @@ def __init__(self, method, horizon): self.horizon = horizon def forward(self, x): + """ + Apply time series normalization to the input tensor. + + This method normalizes the input time series data based on the specified method and horizon. + + Args: + x (torch.Tensor): The input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the feature dimension. + + Returns: + torch.Tensor: The normalized tensor of the same shape as the input. + + Examples: + >>> ts_norm = TSNormalization(method="mean", horizon=24) + >>> input = torch.randn(32, 100, 64) # 32 batches, 100 time steps, 64 features + >>> output = ts_norm(input) + >>> print(output.shape) + torch.Size([32, 100, 64]) + + Note: + - For the "mean" method, it uses the mean of absolute values up to the horizon for normalization. + - For the "last" method, it uses the last value before the horizon for normalization. + - If the method is neither "mean" nor "last", the input is returned unchanged. + """ # x must be BLD if self.method == "mean": self.scale = x.abs()[:, : -self.horizon].mean(dim=1)[:, None, :] @@ -362,6 +765,22 @@ def forward(self, x): class TSInverseNormalization(nn.Module): + """ + A module for inverting time series normalization. + + This class implements the inverse operation of TSNormalization, allowing + the restoration of normalized time series data to its original scale. + + Attributes: + method (str): The normalization method that was used. Supported values are "mean" and "last". + normalizer (TSNormalization): The TSNormalization instance that was used for the initial normalization. + + Note: + This module is typically used in conjunction with TSNormalization to revert + normalized predictions or processed data back to their original scale in + time series analysis and forecasting tasks. + """ + def __init__(self, method, normalizer): super().__init__() @@ -369,12 +788,57 @@ def __init__(self, method, normalizer): self.normalizer = normalizer def forward(self, x): + """ + Apply inverse time series normalization to the input tensor. + + This method reverses the normalization applied by TSNormalization, restoring + the data to its original scale. + + Args: + x (torch.Tensor): The normalized input tensor of shape (B, L, D), where B is the batch size, + L is the sequence length, and D is the feature dimension. + + Returns: + torch.Tensor: The denormalized tensor of the same shape as the input. + + Examples: + >>> ts_norm = TSNormalization(method="mean", horizon=24) + >>> ts_inverse_norm = TSInverseNormalization(method="mean", normalizer=ts_norm) + >>> original_input = torch.randn(32, 100, 64) + >>> normalized = ts_norm(original_input) + >>> denormalized = ts_inverse_norm(normalized) + >>> print(torch.allclose(original_input, denormalized)) + True + + Note: + - For the "mean" and "last" methods, it multiplies the input by the scale stored in the normalizer. + - If the method is neither "mean" nor "last", the input is returned unchanged. + """ if self.method == "mean" or self.method == "last": return x * self.normalizer.scale return x class ReversibleInstanceNorm1dInput(nn.Module): + """ + A reversible instance normalization module for 1D input. + + This module applies instance normalization to the input in a way that allows + for reversing the normalization process later. It is designed to work with + 1D data, such as time series or sequential data. + + Attributes: + transposed (bool): If True, expects input in BDL format, otherwise in BLD format. + norm (nn.InstanceNorm1d): The instance normalization layer. + s (torch.Tensor): Standard deviation of the input, computed during forward pass. + m (torch.Tensor): Mean of the input, computed during forward pass. + + Note: + This module is typically used in pairs with ReversibleInstanceNorm1dOutput + to allow for normalization that can be undone, which can be useful in + certain types of neural network architectures or processing pipelines. + """ + def __init__(self, d, transposed=False): super().__init__() # BLD if transpoed is False, otherwise BDL @@ -382,6 +846,32 @@ def __init__(self, d, transposed=False): self.norm = nn.InstanceNorm1d(d, affine=True, track_running_stats=False) def forward(self, x): + """ + Apply reversible instance normalization to the input tensor. + + This method normalizes the input tensor and stores the statistics needed for + reversing the normalization later. + + Args: + x (torch.Tensor): The input tensor. If transposed is False, expected shape is (B, L, D), + otherwise (B, D, L), where B is batch size, L is sequence length, and D is feature dimension. + + Returns: + torch.Tensor: The normalized tensor with the same shape as the input. + + Examples: + >>> rev_norm = ReversibleInstanceNorm1dInput(64, transposed=False) + >>> input = torch.randn(32, 100, 64) # 32 batches, 100 time steps, 64 features + >>> output = rev_norm(input) + >>> print(output.shape) + torch.Size([32, 100, 64]) + + Note: + - The method computes and stores the mean and standard deviation of the input. + - A small epsilon (1e-4) is added to the standard deviation to avoid division by zero. + - The normalization is applied along the last dimension for non-transposed input, + and along the second-to-last dimension for transposed input. + """ # Means, stds if not self.transposed: x = x.transpose(-1, -2) @@ -398,6 +888,23 @@ def forward(self, x): class ReversibleInstanceNorm1dOutput(nn.Module): + """ + A module for reversing the instance normalization applied by ReversibleInstanceNorm1dInput. + + This module is designed to work in conjunction with ReversibleInstanceNorm1dInput + to undo the normalization process, restoring the data to its original scale and distribution. + + Attributes: + transposed (bool): If True, expects input in BDL format, otherwise in BLD format. + weight (nn.Parameter): The weight parameter from the input normalization module. + bias (nn.Parameter): The bias parameter from the input normalization module. + norm_input (ReversibleInstanceNorm1dInput): The input normalization module used for the forward pass. + + Note: + This module should be used with tensors that have been normalized by a corresponding + ReversibleInstanceNorm1dInput instance to ensure correct denormalization. + """ + def __init__(self, norm_input): super().__init__() self.transposed = norm_input.transposed @@ -406,6 +913,32 @@ def __init__(self, norm_input): self.norm_input = norm_input def forward(self, x): + """ + Reverse the instance normalization applied to the input tensor. + + This method denormalizes the input tensor using the statistics stored in the + corresponding ReversibleInstanceNorm1dInput module. + + Args: + x (torch.Tensor): The normalized input tensor. If transposed is False, expected shape is (B, L, D), + otherwise (B, D, L), where B is batch size, L is sequence length, and D is feature dimension. + + Returns: + torch.Tensor: The denormalized tensor with the same shape as the input. + + Examples: + >>> rev_norm_input = ReversibleInstanceNorm1dInput(64, transposed=False) + >>> rev_norm_output = ReversibleInstanceNorm1dOutput(rev_norm_input) + >>> original = torch.randn(32, 100, 64) + >>> normalized = rev_norm_input(original) + >>> denormalized = rev_norm_output(normalized) + >>> print(torch.allclose(original, denormalized, atol=1e-6)) + True + + Note: + - The denormalization process uses the mean and standard deviation stored in the norm_input attribute. + - If the input was transposed during normalization, it will be transposed again during denormalization. + """ if not self.transposed: x = x.transpose(-1, -2) diff --git a/espnet2/asr/state_spaces/ff.py b/espnet2/asr/state_spaces/ff.py index 67ac605eda6..93615d2cc7f 100644 --- a/espnet2/asr/state_spaces/ff.py +++ b/espnet2/asr/state_spaces/ff.py @@ -11,6 +11,39 @@ class FF(SequenceModule): + """ + Feed-Forward Network (FFN) block in the style of Transformers. + + This class implements a configurable Feed-Forward Network block, commonly used in + Transformer architectures. It consists of two linear layers with an activation + function and optional dropout in between. + + Attributes: + d_output (int): The output dimension of the FFN block. + transposed (bool): If True, the input is expected to be in transposed form. + ff (nn.Sequential): The sequential layers that make up the FFN block. + + Args: + d_input (int): The input dimension. + expand (int, optional): The expansion factor for the hidden dimension. Defaults to 2. + d_output (int, optional): The output dimension. If None, it's set to d_input. Defaults to None. + transposed (bool, optional): Whether the input is transposed. Defaults to False. + activation (str, optional): The activation function to use. Defaults to "gelu". + initializer (callable, optional): The initializer for the linear layers. Defaults to None. + dropout (float, optional): The dropout rate. Defaults to 0.0. + tie_dropout (bool, optional): If True, uses the same dropout mask for all elements. Defaults to False. + + Example: + >>> ffn = FF(d_input=512, expand=4, d_output=512, dropout=0.1) + >>> x = torch.randn(32, 100, 512) # (batch_size, seq_len, d_input) + >>> output, _ = ffn(x) + >>> print(output.shape) + torch.Size([32, 100, 512]) + + Note: + This implementation is derived from the state-spaces repository by Hazy Research. + """ + def __init__( self, d_input, @@ -59,9 +92,64 @@ def __init__( ) def forward(self, x, *args, **kwargs): + """ + Forward pass of the Feed-Forward Network. + + This method applies the feed-forward transformation to the input tensor. + + Args: + x (torch.Tensor): The input tensor. Shape depends on the 'transposed' attribute: + - If transposed=False: [batch_size, seq_len, d_input] + - If transposed=True: [batch_size, d_input, seq_len] + *args: Variable length argument list. Not used in this method but included for compatibility. + **kwargs: Arbitrary keyword arguments. Not used in this method but included for compatibility. + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor after applying the feed-forward transformation. + Shape will be the same as the input tensor, with the last dimension changed to d_output. + - None: Placeholder for consistency with other modules that might return a state. + + Example: + >>> ffn = FF(d_input=512, d_output=512) + >>> x = torch.randn(32, 100, 512) # (batch_size, seq_len, d_input) + >>> output, _ = ffn.forward(x) + >>> print(output.shape) + torch.Size([32, 100, 512]) + """ return self.ff(x), None def step(self, x, state, **kwargs): + """ + Perform a single step of the Feed-Forward Network. + + This method is designed for sequential processing, applying the feed-forward + transformation to a single time step of the input. + + Args: + x (torch.Tensor): The input tensor for a single time step. + - If transposed=False: Shape is [batch_size, d_input] + - If transposed=True: Shape is [batch_size, d_input, 1] + state: Unused parameter, included for compatibility with other sequential modules. + **kwargs: Additional keyword arguments. Not used in this method but included for compatibility. + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor after applying the feed-forward transformation. + Shape will be [batch_size, d_output] if not transposed, or [batch_size, d_output, 1] if transposed. + - Any: The unchanged state parameter, returned for consistency with other sequential modules. + + Example: + >>> ffn = FF(d_input=512, d_output=512, transposed=False) + >>> x = torch.randn(32, 512) # (batch_size, d_input) + >>> output, _ = ffn.step(x, None) + >>> print(output.shape) + torch.Size([32, 512]) + + Note: + The behavior of this method depends on the 'transposed' attribute of the FF class. + When transposed=True, the input is unsqueezed before processing and squeezed afterwards. + """ # x: [batch, d_input] if self.transposed: # expects: [batch, d_input, seq_len] diff --git a/espnet2/asr/state_spaces/model.py b/espnet2/asr/state_spaces/model.py index a962b11a45a..1edf74fae0b 100644 --- a/espnet2/asr/state_spaces/model.py +++ b/espnet2/asr/state_spaces/model.py @@ -13,26 +13,42 @@ class SequenceModel(SequenceModule): - """Isotropic deep sequence model backbone, in the style of ResNets / Transformers. + """ + Isotropic deep sequence model backbone, in the style of ResNets / Transformers. - The SequenceModel class implements a generic - (batch, length, d_input) -> (batch, length, d_output) transformation + This class implements a generic (batch, length, d_input) -> (batch, length, d_output) transformation. Args: - d_model: Resize input (useful for deep models with residuals) - n_layers: Number of layers - transposed: Transpose inputs so each layer receives (batch, dim, length) - dropout: Dropout parameter applied on every residual and every layer - tie_dropout: Tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d - prenorm: Pre-norm vs. post-norm - n_repeat: Each layer is repeated n times per stage before applying pooling - layer: Layer config, must be specified - residual: Residual config - norm: Normalization config (e.g. layer vs batch) - pool: Config for pooling layer per stage - track_norms: Log norms of each layer output - dropinp: Input dropout - drop_path: Stochastic depth for each residual path + d_model (int): Dimension of the model. Used to resize input (useful for deep models with residuals). + n_layers (int, optional): Number of layers. Defaults to 1. + transposed (bool, optional): If True, transpose inputs so each layer receives (batch, dim, length). Defaults to False. + dropout (float, optional): Dropout parameter applied on every residual and every layer. Defaults to 0.0. + tie_dropout (bool, optional): If True, tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d. Defaults to False. + prenorm (bool, optional): If True, use pre-norm instead of post-norm. Defaults to True. + n_repeat (int, optional): Number of times each layer is repeated per stage before applying pooling. Defaults to 1. + layer (dict or list, optional): Layer configuration. Must be specified. + residual (dict, optional): Residual configuration. + norm (dict or str, optional): Normalization configuration (e.g., layer vs batch). + pool (dict, optional): Configuration for pooling layer per stage. + track_norms (bool, optional): If True, log norms of each layer output. Defaults to True. + dropinp (float, optional): Input dropout rate. Defaults to 0.0. + drop_path (float, optional): Stochastic depth rate for each residual path. Defaults to 0.0. + + Attributes: + d_output (int): Output dimension of the model. + layers (nn.ModuleList): List of SequenceResidualBlock modules. + norm (nn.Module): Normalization layer (if prenorm is True) or nn.Identity. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4, dropout=0.1) + >>> inputs = torch.randn(32, 100, 256) # (batch, length, d_input) + >>> outputs, _ = model(inputs) + >>> outputs.shape + torch.Size([32, 100, 256]) + + Note: + The model can handle both transposed and non-transposed inputs, depending on the 'transposed' parameter. + It also supports various configurations for normalization, residual connections, and pooling. """ def __init__( @@ -117,6 +133,40 @@ def __init__( self.norm = nn.Identity() def forward(self, inputs, *args, state=None, **kwargs): + """ + Forward pass of the SequenceModel. + + This method processes the input through all layers of the model, applying + dropout, normalization, and tracking norms if specified. + + Args: + inputs (torch.Tensor): Input tensor of shape (batch, sequence, dim) or + (batch, dim, sequence) if self.transposed is True. + *args: Variable length argument list. + state (list, optional): List of previous states for each layer. If None, + default states will be used. Defaults to None. + **kwargs: Arbitrary keyword arguments. + + Returns: + tuple: A tuple containing: + - outputs (torch.Tensor): The processed output tensor of shape + (batch, sequence, d_output) or (batch, d_output, sequence) if + self.transposed is True. + - next_states (list): List of updated states for each layer. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4) + >>> inputs = torch.randn(32, 100, 256) + >>> outputs, states = model.forward(inputs) + >>> outputs.shape + torch.Size([32, 100, 256]) + >>> len(states) + 4 + + Note: + If self.track_norms is True, the method will update self.metrics with + the mean squared values of the outputs at each layer. + """ # Inputs assumed to be (batch, sequence, dim) if self.transposed: inputs = rearrange(inputs, "b ... d -> b d ...") @@ -149,11 +199,57 @@ def forward(self, inputs, *args, state=None, **kwargs): @property def d_state(self): + """ + Total dimension of the state across all layers. + + This property calculates and returns the sum of state dimensions for all layers + in the model that have a state. + + Returns: + int: The total dimension of the state across all layers. If a layer doesn't + have a state (i.e., its d_state is None), it's not included in the sum. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4) + >>> model.d_state + 1024 # Assuming each layer has a state dimension of 256 + + Note: + This property is useful for understanding the total state size of the model, + which can be important for memory considerations or when initializing or + processing the entire state of the model at once. + """ d_states = [layer.d_state for layer in self.layers] return sum([d for d in d_states if d is not None]) @property def state_to_tensor(self): + """ + Property that returns a function to convert model state to a tensor. + + This property provides a function that concatenates the tensor representations + of states from all layers into a single tensor. + + Returns: + function: A function that takes a state (list of layer states) as input + and returns a concatenated tensor of all non-None states. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4) + >>> state = model.default_state(batch_size=32) + >>> state_tensor = model.state_to_tensor(state) + >>> state_tensor.shape + torch.Size([32, 1024]) # Assuming each layer has a state dimension of 256 + + Note: + This property uses a closure to create a function that has access to + the model's layers. The returned function is "curried" in the sense that + it's partially applied to the model's layers and can be called later + with a state argument. + + The function skips any None states, only concatenating tensor states. + """ + # Slightly hacky way to implement this in a curried manner # (so that the function can be extracted from an instance) # Somewhat more sound may be to turn this into a @@ -169,11 +265,75 @@ def fn(state): return fn def default_state(self, *batch_shape, device=None): + """ + Generate the default initial state for the model. + + This method creates a list of default states for each layer in the model. + + Args: + *batch_shape (tuple): Variable length argument list for the batch dimensions. + device (torch.device, optional): The device on which to create the state tensors. + If None, uses the default device. Defaults to None. + + Returns: + list: A list of default states for each layer in the model. The structure and + content of each state depends on the specific implementation of each layer. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4) + >>> state = model.default_state(32) # for a batch size of 32 + >>> len(state) + 4 # one state per layer + >>> state = model.default_state(32, device=torch.device('cuda')) # on GPU + + Note: + The shape and content of the default state for each layer may vary depending + on the layer type and configuration. Some layers might return None if they + don't maintain a state. + + This method is particularly useful for initializing the model's state at the + beginning of a sequence or when no previous state is available. + """ return [ layer.default_state(*batch_shape, device=device) for layer in self.layers ] def step(self, x, state, **kwargs): + """ + Perform a single step forward pass of the model. + + This method applies the model to a single step of input, updating the state + for each layer in the process. + + Args: + x (torch.Tensor): Input tensor for a single step. The shape should be + compatible with the model's input requirements for a single time step. + state (list, optional): List of previous states for each layer. If None, + default states will be used. Defaults to None. + **kwargs: Additional keyword arguments to be passed to each layer's step method. + + Returns: + tuple: A tuple containing: + - x (torch.Tensor): The output tensor after processing through all layers. + - next_states (list): List of updated states for each layer. + + Examples: + >>> model = SequenceModel(d_model=256, n_layers=4) + >>> x = torch.randn(32, 256) # (batch_size, d_model) + >>> state = model.default_state(32) + >>> output, new_state = model.step(x, state) + >>> output.shape + torch.Size([32, 256]) + >>> len(new_state) + 4 + + Note: + This method is particularly useful for processing sequences one step at a time, + which can be more memory-efficient for very long sequences or in scenarios + where future inputs are not available, such as in real-time or streaming applications. + + The final normalization layer is applied to the output before returning. + """ # Apply layers prev_states = [None] * len(self.layers) if state is None else state next_states = [] diff --git a/espnet2/asr/state_spaces/pool.py b/espnet2/asr/state_spaces/pool.py index 4d08194d770..dc4e6f4e293 100644 --- a/espnet2/asr/state_spaces/pool.py +++ b/espnet2/asr/state_spaces/pool.py @@ -18,6 +18,35 @@ def downsample(x, stride=1, expand=1, transposed=False): + """ + Downsample a sequence along the layer dimension and optionally expand along the feature dimension. + + This function performs downsampling on the input sequence by selecting elements at regular intervals + and optionally expands the feature dimension by repeating values. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + stride (int, optional): The stride for downsampling. Defaults to 1. + expand (int, optional): The expansion factor for the feature dimension. Defaults to 1. + transposed (bool, optional): Whether the input is in transposed format. Defaults to False. + + Returns: + torch.Tensor: Downsampled and optionally expanded tensor. + + Raises: + AssertionError: If the input tensor is not 3-dimensional when stride > 1. + + Examples: + >>> x = torch.randn(32, 100, 64) # (batch_size, seq_len, feature_dim) + >>> y = downsample(x, stride=2, expand=2) + >>> y.shape + torch.Size([32, 50, 128]) + + Note: + - If stride > 1, the function only supports 3-dimensional inputs. + - For higher-dimensional inputs with stride > 1, it is recommended to use average or spectral pooling instead. + """ if x is None: return None if stride > 1: @@ -40,6 +69,32 @@ def downsample(x, stride=1, expand=1, transposed=False): def upsample(x, stride=1, expand=1, transposed=False): + """ + Upsample a sequence along the layer dimension and optionally reduce along the feature dimension. + + This function performs upsampling on the input sequence by repeating elements and optionally + reduces the feature dimension by averaging values. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + stride (int, optional): The stride for upsampling. Defaults to 1. + expand (int, optional): The reduction factor for the feature dimension. Defaults to 1. + transposed (bool, optional): Whether the input is in transposed format. Defaults to False. + + Returns: + torch.Tensor: Upsampled and optionally reduced tensor. + + Examples: + >>> x = torch.randn(32, 50, 128) # (batch_size, seq_len, feature_dim) + >>> y = upsample(x, stride=2, expand=2) + >>> y.shape + torch.Size([32, 100, 64]) + + Note: + - If expand > 1, the function reduces the feature dimension by taking the mean of groups of 'expand' features. + - If stride > 1, the function repeats each element 'stride' times along the sequence dimension. + """ if x is None: return None if expand > 1: @@ -56,6 +111,23 @@ def upsample(x, stride=1, expand=1, transposed=False): class DownSample(SequenceModule): + """ + A module for downsampling sequences. + + This class implements downsampling on input sequences by selecting elements at regular intervals + and optionally expanding the feature dimension. It inherits from SequenceModule and can be used + in sequential models. + + Attributes: + d_input (int): The input feature dimension. + stride (int): The stride for downsampling. + expand (int): The expansion factor for the feature dimension. + transposed (bool): Whether the input is in transposed format. + + Note: + The `step` method is not implemented for stride > 1 or expand > 1. + """ + def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -64,19 +136,90 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): + """ + Forward pass for the DownSample module. + + This method applies downsampling to the input tensor using the specified stride and expand parameters. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + + Returns: + torch.Tensor: Downsampled tensor with shape depending on the stride and expand parameters. + + Note: + This method uses the `downsample` function internally, with `transposed=False` hardcoded. + The actual transposition behavior is controlled by the `self.transposed` attribute. + """ return downsample(x, self.stride, self.expand, False, self.transposed) def step(self, x, state, **kwargs): + """ + Perform a single step of the DownSample module. + + This method is intended for use in sequential or recurrent scenarios where the input is processed + one step at a time. + + Args: + x (torch.Tensor): Input tensor for the current step. + state: The current state of the module (unused in this implementation). + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor for the current step (same as input). + - state: The updated state (unchanged in this implementation). + + Raises: + NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. + + Note: + This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. + """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): + """ + int: The output feature dimension of the DownSample module. + + This property calculates and returns the output feature dimension based on the input dimension + and the expansion factor. + + Returns: + int: The output feature dimension, which is the product of the input dimension and the expand factor. + + Note: + This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. + """ return self.d_input * self.expand class DownAvgPool(SequenceModule): + """ + A module for downsampling sequences using average pooling. + + This class implements downsampling on input sequences by applying average pooling + and optionally expanding the feature dimension. It inherits from SequenceModule and + can be used in sequential models. + + Attributes: + d_input (int): The input feature dimension. + stride (int): The stride for downsampling. + expand (int): The expansion factor for the feature dimension. + transposed (bool): Whether the input is in transposed format. + + Note: + - This module supports multi-dimensional inputs (e.g., 1D, 2D, or higher). + - The `step` method is not implemented for stride > 1 or expand > 1. + - Average pooling is applied along the sequence dimension(s) when stride > 1. + - Feature expansion is performed by repeating when expand > 1. + """ + def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -85,6 +228,26 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): + """ + Forward pass for the DownAvgPool module. + + This method applies average pooling for downsampling and optionally expands the feature dimension + of the input tensor. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, ..., feature_dim) if not transposed, + or (batch_size, feature_dim, ...) if transposed. + + Returns: + torch.Tensor: Downsampled and optionally expanded tensor. + + Note: + - If not transposed, the input is rearranged to (batch_size, feature_dim, ...) before processing. + - Average pooling is applied using F.avg_pool1d for 3D inputs, F.avg_pool2d for 4D inputs, + and a custom reduction for higher dimensions. + - If expand > 1, the feature dimension is expanded by repeating. + - The output is rearranged back to the original format if not transposed. + """ if not self.transposed: x = rearrange(x, "b ... d -> b d ...") @@ -112,16 +275,75 @@ def forward(self, x): return x def step(self, x, state, **kwargs): + """ + Perform a single step of the DownAvgPool module. + + This method is intended for use in sequential or recurrent scenarios where the input is processed + one step at a time. + + Args: + x (torch.Tensor): Input tensor for the current step. + state: The current state of the module (unused in this implementation). + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor for the current step (same as input). + - state: The updated state (unchanged in this implementation). + + Raises: + NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. + + Note: + This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. + """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): + """ + int: The output feature dimension of the DownAvgPool module. + + This property calculates and returns the output feature dimension based on the input dimension + and the expansion factor. + + Returns: + int: The output feature dimension, which is the product of the input dimension and the expand factor. + + Note: + This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. The output dimension is expanded + if the `expand` attribute is greater than 1. + """ return self.d_input * self.expand class DownSpectralPool(SequenceModule): + """ + A module for downsampling sequences using spectral pooling. + + This class implements downsampling on input sequences by applying spectral pooling + in the frequency domain and optionally expanding the feature dimension. It inherits + from SequenceModule and can be used in sequential models. + + Spectral pooling is performed by truncating high-frequency components in the Fourier domain, + which can preserve more information compared to traditional spatial pooling methods. + + Attributes: + d_input (int): The input feature dimension. + stride (int): The stride for downsampling. + expand (int): The expansion factor for the feature dimension. + transposed (bool): Whether the input is in transposed format. + + Note: + - This module supports multi-dimensional inputs. + - The `step` method is not implemented for stride > 1 or expand > 1. + - Spectral pooling is applied along all spatial dimensions when stride > 1. + - Feature expansion is performed by repeating when expand > 1. + """ + def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -130,9 +352,30 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): - """Forward pass. + """ + Forward pass for the DownSpectralPool module. + + This method applies spectral pooling for downsampling and optionally expands the feature dimension + of the input tensor. The spectral pooling is performed in the frequency domain using FFT. + + Args: + x (torch.Tensor): Input tensor of shape (B, L..., D) where B is batch size, L... are spatial dimensions, + and D is the feature dimension. + + Returns: + torch.Tensor: Downsampled and optionally expanded tensor. + + Raises: + AssertionError: If any spatial dimension length is not divisible by the stride. - x: (B, L..., D) + Note: + - If not transposed, the input is rearranged to (B, D, L...) before processing. + - The method performs the following steps: + 1. Applies FFT to the input tensor. + 2. Truncates high-frequency components based on the stride. + 3. Applies inverse FFT to return to the spatial domain. + 4. Expands the feature dimension if expand > 1. + - The output is rearranged back to the original format if not transposed. """ if not self.transposed: x = rearrange(x, "b ... d -> b d ...") @@ -156,16 +399,73 @@ def forward(self, x): return x def step(self, x, state, **kwargs): + """ + Perform a single step of the DownSpectralPool module. + + This method is intended for use in sequential or recurrent scenarios where the input is processed + one step at a time. + + Args: + x (torch.Tensor): Input tensor for the current step. + state: The current state of the module (unused in this implementation). + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor for the current step (same as input). + - state: The updated state (unchanged in this implementation). + + Raises: + NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. + + Note: + This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. + Spectral pooling is not performed in the step method, as it requires processing the entire sequence. + """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): + """ + int: The output feature dimension of the DownSpectralPool module. + + This property calculates and returns the output feature dimension based on the input dimension + and the expansion factor. + + Returns: + int: The output feature dimension, which is the product of the input dimension and the expand factor. + + Note: + This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. The output dimension is expanded + if the `expand` attribute is greater than 1, while the spatial dimensions are reduced based + on the `stride` attribute. + """ return self.d_input * self.expand class UpSample(nn.Module): + """ + A module for upsampling sequences. + + This class implements upsampling on input sequences by repeating elements along the sequence dimension + and optionally reducing the feature dimension. It inherits from nn.Module and can be used in + neural network architectures. + + Attributes: + d_input (int): The input feature dimension. + stride (int): The stride for upsampling (number of times each element is repeated). + expand (int): The reduction factor for the feature dimension. + transposed (bool): Whether the input is in transposed format. + + Note: + - The actual upsampling is performed in the forward method using the `upsample` function. + - The `step` method is not implemented for stride > 1 or expand > 1. + - The output dimension is calculated as d_input // expand, effectively reducing the feature dimension. + """ + def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -174,13 +474,69 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): + """ + Forward pass for the UpSample module. + + This method applies upsampling to the input tensor using the specified stride and expand parameters. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + + Returns: + torch.Tensor: Upsampled tensor with shape depending on the stride and expand parameters. + + Note: + This method uses the `upsample` function internally. The upsampling process involves: + 1. Repeating elements along the sequence dimension based on the `stride` parameter. + 2. Reducing the feature dimension based on the `expand` parameter (if > 1). + The exact behavior depends on whether the input is in transposed format or not. + """ return upsample(x, self.stride, self.expand, self.transposed) @property def d_output(self): + """ + int: The output feature dimension of the UpSample module. + + This property calculates and returns the output feature dimension based on the input dimension + and the expand factor. + + Returns: + int: The output feature dimension, which is the input dimension divided by the expand factor. + + Note: + This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. The output dimension is reduced + if the `expand` attribute is greater than 1, reflecting the feature dimension reduction + that occurs during upsampling. + """ return self.d_input // self.expand def step(self, x, state, **kwargs): + """ + Perform a single step of the UpSample module. + + This method is intended for use in sequential or recurrent scenarios where the input is processed + one step at a time. + + Args: + x (torch.Tensor): Input tensor for the current step. + state: The current state of the module (unused in this implementation). + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor for the current step (same as input). + - state: The updated state (unchanged in this implementation). + + Raises: + NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. + + Note: + This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. + Upsampling is not performed in the step method, as it requires processing multiple time steps at once. + """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @@ -191,6 +547,26 @@ def step(self, x, state, **kwargs): class DownLinearPool(SequenceModule): + """ + A module for downsampling sequences using linear pooling. + + This class implements downsampling on input sequences by applying a linear transformation + to groups of elements and optionally expanding the feature dimension. It inherits from + SequenceModule and can be used in sequential models. + + Attributes: + d_input (int): The input feature dimension. + stride (int): The stride for downsampling (number of elements to group). + expand (int): The expansion factor for the feature dimension. + transposed (bool): Whether the input is in transposed format. + linear (LinearActivation): The linear transformation applied to grouped elements. + + Note: + - The linear transformation is applied to groups of `stride` elements along the sequence dimension. + - The output feature dimension is expanded by a factor of `expand`. + - The `step` method is not implemented for stride > 1 or expand > 1. + """ + def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() @@ -206,6 +582,25 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): ) def forward(self, x): + """ + Forward pass for the DownLinearPool module. + + This method applies linear pooling for downsampling and optionally expands the feature dimension + of the input tensor. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + + Returns: + torch.Tensor: Downsampled and optionally expanded tensor. + + Note: + - If transposed, the input is rearranged to group `stride` elements along the last dimension. + - If not transposed, the input is rearranged to group `stride` elements along the second-to-last dimension. + - The linear transformation is applied to the grouped elements, effectively downsampling the sequence. + - The output tensor's shape depends on the stride, expand, and transposed parameters. + """ if self.transposed: x = rearrange(x, "... h (l s) -> ... (h s) l", s=self.stride) else: @@ -214,12 +609,50 @@ def forward(self, x): return x def step(self, x, state, **kwargs): + """ + Perform a single step of the DownLinearPool module. + + This method is intended for use in sequential or recurrent scenarios where the input is processed + one step at a time. + + Args: + x (torch.Tensor): Input tensor for the current step. + state: The current state of the module (unused in this implementation). + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: The output tensor for the current step (same as input). + - state: The updated state (unchanged in this implementation). + + Raises: + NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. + + Note: + This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. + Linear pooling is not performed in the step method, as it requires processing multiple time steps at once. + """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): + """ + int: The output feature dimension of the DownLinearPool module. + + This property calculates and returns the output feature dimension based on the input dimension + and the expansion factor. + + Returns: + int: The output feature dimension, which is the product of the input dimension and the expand factor. + + Note: + This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. The output dimension is expanded + by the `expand` factor, reflecting the feature dimension expansion that occurs during + the linear pooling operation. + """ return self.d_input * self.expand @@ -227,6 +660,23 @@ def d_output(self): class DownPool2d(SequenceModule): + """ + A module for downsampling 2D sequences using a combination of linear transformation and average pooling. + + This class implements downsampling on 2D input sequences by first applying a linear transformation + to change the feature dimension, followed by average pooling to reduce spatial dimensions. It inherits + from SequenceModule and can be used in sequential models dealing with 2D data. + + Attributes: + linear (LinearActivation): Linear transformation applied to the input before pooling. + pool (nn.AvgPool2d): Average pooling layer for spatial downsampling. + + Note: + - The linear transformation allows for changing the feature dimension before pooling. + - The average pooling operation reduces the spatial dimensions based on the given stride. + - This module is particularly useful for processing 2D data such as images or feature maps. + """ + def __init__(self, d_input, d_output, stride=1, transposed=True, weight_norm=True): super().__init__() @@ -240,6 +690,23 @@ def __init__(self, d_input, d_output, stride=1, transposed=True, weight_norm=Tru self.pool = (nn.AvgPool2d(kernel_size=stride, stride=stride),) def forward(self, x): + """ + Forward pass for the DownPool2d module. + + This method applies the linear transformation followed by average pooling to downsample the input tensor. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, channels, height, width) if transposed, + or (batch_size, height, width, channels) if not transposed. + + Returns: + torch.Tensor: Downsampled tensor after linear transformation and pooling. + + Note: + - If transposed, the pooling operation is applied directly after the linear transformation. + - The implementation for non-transposed input is not provided in the given code snippet. + - The output tensor's spatial dimensions will be reduced based on the pooling parameters. + """ if self.transposed: x = self.pool(x) @@ -247,6 +714,26 @@ def forward(self, x): # DownLinearPool is used by the registry (for isotropic backbone) # DownPool is essentially the same as DownLinearPool. These should be consolidated class DownPool(SequenceModule): + """ + A flexible module for downsampling sequences using linear transformation. + + This class implements downsampling on input sequences by applying a linear transformation + to groups of elements. It allows for both downsampling and feature dimension adjustment. + It inherits from SequenceModule and can be used in sequential models. + + Attributes: + d_output (int): The output feature dimension. + stride (int): The stride for downsampling (number of elements to group). + transposed (bool): Whether the input is in transposed format. + linear (LinearActivation): The linear transformation applied to grouped elements. + + Note: + - The linear transformation is applied to groups of `stride` elements along the sequence dimension. + - This module can both downsample the sequence and adjust the feature dimension. + - Supports optional weight normalization and activation function in the linear transformation. + - The `step` method provides functionality for recurrent processing. + """ + def __init__( self, d_input, @@ -278,6 +765,26 @@ def __init__( ) def forward(self, x): + """ + Forward pass for the DownPool module. + + This method applies downsampling and linear transformation to the input tensor. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Downsampled and transformed tensor. + - None: Placeholder for consistency with other modules that might return additional information. + + Note: + - If transposed, the input is rearranged to group `stride` elements along the second dimension. + - If not transposed, the input is rearranged to group `stride` elements along the second-to-last dimension. + - The linear transformation is applied to the grouped elements, effectively downsampling the sequence. + - The output tensor's shape depends on the stride, d_output, and transposed parameters. + """ if self.transposed: x = rearrange(x, "... h (l s) -> ... (h s) l", s=self.stride) else: @@ -286,9 +793,27 @@ def forward(self, x): return x, None def step(self, x, state, **kwargs): - """Step one time step as a recurrent model. - - x: (..., H) + """ + Perform a single step of the DownPool module for recurrent processing. + + This method allows the DownPool module to be used in a recurrent manner, processing + one time step at a time and maintaining an internal state. + + Args: + x (torch.Tensor or None): Input tensor for the current step, or None if no input is available. + state (list): Current state of the module, containing previously seen inputs. + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor or None: Output tensor if a full group is processed, otherwise None. + - list: Updated state of the module. + + Note: + - The method accumulates inputs in the state until it has gathered 'stride' elements. + - When 'stride' elements are gathered, it applies the linear transformation and resets the state. + - If x is None, it returns None and the current state. + - This method enables the module to work with variable-length sequences in a recurrent setting. """ if x is None: return None, state @@ -305,10 +830,49 @@ def step(self, x, state, **kwargs): return None, state def default_state(self, *batch_shape, device=None): + """ + Initialize the default state for the DownPool module. + + This method creates the initial state for the module when used in a recurrent setting. + + Args: + *batch_shape: Variable length argument to specify the batch dimensions. + device (torch.device, optional): The device on which to create the state. Defaults to None. + + Returns: + list: An empty list representing the initial state of the module. + + Note: + - The default state is an empty list, which will be filled with input tensors during the step method. + - This method is typically called once before starting the recurrent processing of a sequence. + - The batch_shape and device arguments are included for compatibility with other modules + but are not used in this implementation. + """ return [] class UpPool(SequenceModule): + """ + A module for upsampling sequences using linear transformation. + + This class implements upsampling on input sequences by applying a linear transformation + followed by reshaping to increase the sequence length. It inherits from SequenceModule + and can be used in sequential models. + + Attributes: + d_input (int): The input feature dimension. + _d_output (int): The output feature dimension. + stride (int): The upsampling factor (increase in sequence length). + transposed (bool): Whether the input is in transposed format. + linear (LinearActivation): The linear transformation applied before upsampling. + + Note: + - The linear transformation expands the feature dimension by a factor of `stride`. + - The output is then reshaped to increase the sequence length by `stride`. + - Supports optional weight normalization and activation function in the linear transformation. + - Includes a step method for recurrent processing and a default_state method for initialization. + """ + def __init__( self, d_input, @@ -337,6 +901,28 @@ def __init__( ) def forward(self, x, skip=None): + """ + Forward pass for the UpPool module. + + This method applies upsampling to the input tensor using linear transformation and reshaping. + + Args: + x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, + or (batch_size, feature_dim, seq_len) if transposed. + skip (torch.Tensor, optional): Skip connection input to be added after upsampling. Defaults to None. + + Returns: + tuple: A tuple containing: + - torch.Tensor: Upsampled tensor, potentially with skip connection added. + - None: Placeholder for consistency with other modules that might return additional information. + + Note: + - The input is first passed through a linear transformation to expand the feature dimension. + - The transformed tensor is then reshaped to increase the sequence length by the stride factor. + - A shift operation is applied to ensure causality in the upsampled sequence. + - If a skip connection is provided, it's added to the upsampled tensor. + - The output tensor's shape depends on the stride, d_output, and transposed parameters. + """ x = self.linear(x) if self.transposed: x = F.pad(x[..., :-1], (1, 0)) # Shift to ensure causality @@ -349,9 +935,27 @@ def forward(self, x, skip=None): return x, None def step(self, x, state, **kwargs): - """Step one time step as a recurrent model. - - x: (..., H) + """ + Perform a single step of the UpPool module for recurrent processing. + + This method allows the UpPool module to be used in a recurrent manner, processing + one time step at a time and maintaining an internal state. + + Args: + x (torch.Tensor or None): Input tensor for the current step, or None if no input is available. + state (list): Current state of the module, containing future outputs. + **kwargs: Additional keyword arguments (unused in this implementation). + + Returns: + tuple: A tuple containing: + - torch.Tensor: Output tensor for the current step. + - list: Updated state of the module. + + Note: + - The method returns the next output from the state and updates the state. + - If the state is empty and x is provided, it applies the linear transformation and generates new outputs. + - This method enables the module to work with variable-length sequences in a recurrent setting. + - It maintains causality by only using current and past inputs to generate outputs. """ assert len(state) > 0 y, state = state[0], state[1:] @@ -369,6 +973,25 @@ def step(self, x, state, **kwargs): return y, state def default_state(self, *batch_shape, device=None): + """ + Initialize the default state for the UpPool module. + + This method creates the initial state for the module when used in a recurrent setting. + + Args: + *batch_shape: Variable length argument to specify the batch dimensions. + device (torch.device, optional): The device on which to create the state. Defaults to None. + + Returns: + list: A list of torch.Tensor objects representing the initial state of the module. + + Note: + - The default state is a list of zero tensors, with shape (batch_shape, d_output, stride). + - Each tensor in the list represents a future output in the upsampled sequence. + - The state is initialized with 'stride' number of zero tensors. + - This method is typically called once before starting the recurrent processing of a sequence. + - The state allows the module to maintain the expanded sequence length during step-wise processing. + """ state = torch.zeros( batch_shape + (self.d_output, self.stride), device=device ) # (batch, h, s) @@ -377,6 +1000,20 @@ def default_state(self, *batch_shape, device=None): @property def d_output(self): + """ + int: The output feature dimension of the UpPool module. + + This property returns the output feature dimension of the module. + + Returns: + int: The output feature dimension. + + Note: + - This property returns the value of the `_d_output` attribute, which is set during initialization. + - The output dimension is typically different from the input dimension due to the upsampling process. + - This property is useful for determining the output shape of the module, especially when + chaining multiple modules together in a sequential model. + """ return self._d_output diff --git a/espnet2/asr/state_spaces/residual.py b/espnet2/asr/state_spaces/residual.py index bcadadb02f1..472bf901c9d 100644 --- a/espnet2/asr/state_spaces/residual.py +++ b/espnet2/asr/state_spaces/residual.py @@ -7,9 +7,37 @@ class Residual(nn.Module): - """Residual connection with constant affine weights. + """ + Residual connection with constant affine weights. + + This class implements a residual connection that can simulate standard residual, + no residual, and "constant gates" behaviors. + + Attributes: + i_layer (int): The index of the current layer. + d_input (int): The dimension of the input. + d_model (int): The dimension of the model. + alpha (float): The weight for the input in the residual connection. + beta (float): The weight for the transformed input in the residual connection. + + Args: + i_layer (int): The index of the current layer. + d_input (int): The dimension of the input. + d_model (int): The dimension of the model. + alpha (float, optional): The weight for the input. Defaults to 1.0. + beta (float, optional): The weight for the transformed input. Defaults to 1.0. - Can simulate standard residual, no residual, and "constant gates". + Note: + The input dimension (d_input) must be equal to the model dimension (d_model) + unless alpha is set to 0.0. + + Examples: + >>> residual = Residual(1, 64, 64) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = residual(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) """ def __init__(self, i_layer, d_input, d_model, alpha=1.0, beta=1.0): @@ -24,18 +52,89 @@ def __init__(self, i_layer, d_input, d_model, alpha=1.0, beta=1.0): @property def d_output(self): + """ + Returns the output dimension of the residual connection. + + Returns: + int: The dimension of the model (d_model), which is the output dimension + of the residual connection. + + Note: + This property always returns the d_model attribute, as the residual + connection maintains the same dimension as the input model. + + Examples: + >>> residual = Residual(1, 64, 64) + >>> residual.d_output + 64 + """ return self.d_model def forward(self, x, y, transposed): + """ + Performs the forward pass of the residual connection. + + This method applies the residual connection by combining the input x + and the transformed input y using the weights alpha and beta. + + Args: + x (torch.Tensor): The input tensor. + y (torch.Tensor): The transformed input tensor. + transposed (bool): A flag indicating whether the input is transposed. + This argument is not used in the current implementation but is + included for consistency with other similar methods. + + Returns: + torch.Tensor: The output of the residual connection. + + Note: + The formula used is: output = alpha * x + beta * y + If beta is 1.0, y is used as is. If alpha is 0, only y is returned. + + Examples: + >>> residual = Residual(1, 64, 64, alpha=0.5, beta=1.5) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = residual.forward(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ y = self.beta * y if self.beta != 1.0 else y return self.alpha * x + y if self.alpha else y class Affine(Residual): - """Residual connection with learnable scalar multipliers on the main branch. + """ + Residual connection with learnable scalar multipliers on the main branch. + + This class extends the Residual class by adding learnable scalar multipliers + to the main branch of the residual connection. It can be initialized with + either a single scalar multiplier or one per dimension. - scalar: Single scalar multiplier, or one per dimension - scale, power: Initialize to scale * layer_num**(-power) + Attributes: + scalar (bool): If True, use a single scalar multiplier; otherwise, use one per dimension. + gamma (float): Power factor for layer-dependent scaling initialization. + affine (nn.Parameter): Learnable scalar multiplier(s) for the main branch. + + Args: + *args: Variable length argument list passed to the parent Residual class. + scalar (bool, optional): Whether to use a single scalar multiplier. Defaults to True. + gamma (float, optional): Power factor for layer-dependent scaling. Defaults to 0.0. + **kwargs: Arbitrary keyword arguments passed to the parent Residual class. + + Note: + The affine parameter is initialized as: + c * torch.ones(d), where + c = beta * i_layer ** (-gamma) + d = 1 if scalar is True, else d_input + + Examples: + >>> affine_residual = Affine(1, 64, 64, scalar=False, gamma=0.1) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = affine_residual(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) """ def __init__(self, *args, scalar=True, gamma=0.0, **kwargs): @@ -49,6 +148,33 @@ def __init__(self, *args, scalar=True, gamma=0.0, **kwargs): self.affine = nn.Parameter(c * torch.ones(d)) def forward(self, x, y, transposed): + """ + Performs the forward pass of the Affine residual connection. + + This method applies the residual connection by combining the input x + and the transformed input y using the learnable affine parameter. + + Args: + x (torch.Tensor): The input tensor. + y (torch.Tensor): The transformed input tensor. + transposed (bool): A flag indicating whether the input is transposed. + If True, the affine parameter is unsqueezed along the last dimension. + + Returns: + torch.Tensor: The output of the Affine residual connection. + + Note: + The formula used is: output = alpha * x + c * y + where c is the learnable affine parameter. + + Examples: + >>> affine_residual = Affine(1, 64, 64, scalar=False, gamma=0.1) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = affine_residual.forward(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ c = self.affine if transposed: c = c.unsqueeze(-1) @@ -56,12 +182,72 @@ def forward(self, x, y, transposed): class Feedforward(Residual): + """ + A feedforward residual connection. + + This class implements a simple feedforward residual connection where the + input x is ignored, and only the transformed input y is passed through. + It's a special case of the Residual class with alpha set to 0.0 and beta set to 1.0. + + Args: + *args: Variable length argument list passed to the parent Residual class. + + Note: + This class effectively removes the residual connection, as it only + passes through the transformed input y without adding the original input x. + + Examples: + >>> feedforward = Feedforward(1, 64, 64) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = feedforward(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + >>> torch.allclose(output, y) + True + """ + def __init__(self, *args): # print("Feedforward extra kwargs", kwargs) super().__init__(*args, alpha=0.0, beta=1.0) class Highway(Residual): + """ + Implements a Highway residual connection. + + This class extends the Residual class to create a Highway network connection, + which allows the model to adaptively choose between passing information + through a nonlinear transformation or passing it unchanged. + + Attributes: + scaling_correction (float): A scaling factor to adjust the output. + elemwise (bool): If True, uses element-wise multiplication for Wy; + otherwise, uses a linear transformation. + Wx (nn.Linear): Linear transformation for the input x. + Wy (nn.Parameter or nn.Linear): Transformation for the input y, + either element-wise or linear. + + Args: + *args: Variable length argument list passed to the parent Residual class. + scaling_correction (bool, optional): If True, applies a scaling correction + factor of 1.732. Defaults to False. + elemwise (bool, optional): If True, uses element-wise multiplication + for Wy. Defaults to False. + + Note: + The Highway connection computes a gating mechanism to control + information flow from the input and transformed input. + + Examples: + >>> highway = Highway(1, 64, 64, scaling_correction=True, elemwise=False) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = highway(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ + def __init__(self, *args, scaling_correction=False, elemwise=False): super().__init__(*args) self.scaling_correction = 1.732 if scaling_correction else 1.0 @@ -73,6 +259,38 @@ def __init__(self, *args, scaling_correction=False, elemwise=False): self.Wy = nn.Linear(self.d_input, self.d_input) def forward(self, x, y, transposed=False): + """ + Performs the forward pass of the Highway residual connection. + + This method implements the Highway network mechanism, which adaptively + combines the original input and its transformation using a gating function. + + Args: + x (torch.Tensor): The original input tensor. + y (torch.Tensor): The transformed input tensor. + transposed (bool): A flag indicating whether the input is transposed. + This argument is not used in the current implementation but is + included for consistency with other similar methods. + + Returns: + torch.Tensor: The output of the Highway residual connection. + + Note: + The Highway mechanism is computed as follows: + 1. Compute the gate: r = sigmoid(Wx(x) + Wy(y)) + 2. Combine inputs: z = scaling_correction * (1 - r) * x + r * y + + If elemwise is True, Wy is applied as element-wise multiplication. + Otherwise, it's applied as a linear transformation. + + Examples: + >>> highway = Highway(1, 64, 64, scaling_correction=True, elemwise=False) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = highway.forward(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ if self.elemwise: y = self.Wy * y else: @@ -83,7 +301,34 @@ def forward(self, x, y, transposed=False): class DecayResidual(Residual): - """Residual connection that can decay the linear combination depending on depth.""" + """ + Implements a residual connection with depth-dependent decay. + + This class extends the Residual class to create a connection where the + contribution of the residual path decays based on the depth of the layer. + + Attributes: + power (float): The power factor used in the decay calculation. + l2 (bool): If True, uses L2 normalization for alpha calculation; + otherwise, uses linear decay. + + Args: + *args: Variable length argument list passed to the parent Residual class. + power (float, optional): The power factor for decay calculation. Defaults to 0.5. + l2 (bool, optional): Whether to use L2 normalization. Defaults to True. + + Note: + The decay is calculated based on the layer index (i_layer) and the power factor. + The contribution of the original input decreases as the network depth increases. + + Examples: + >>> decay_residual = DecayResidual(1, 64, 64, power=0.5, l2=True) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = decay_residual(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ def __init__(self, *args, power=0.5, l2=True): # print("DecayResidual extra kwargs", kwargs) @@ -92,6 +337,42 @@ def __init__(self, *args, power=0.5, l2=True): self.l2 = l2 def forward(self, x, y, transposed): + """ + Performs the forward pass of the DecayResidual connection. + + This method implements the depth-dependent decay mechanism for combining + the original input and its transformation. + + Args: + x (torch.Tensor): The original input tensor. + y (torch.Tensor): The transformed input tensor. + transposed (bool): A flag indicating whether the input is transposed. + This argument is not used in the current implementation but is + included for consistency with other similar methods. + + Returns: + torch.Tensor: The output of the DecayResidual connection. + + Note: + The decay mechanism is computed as follows: + 1. Calculate beta: beta = i_layer ** (-power) + 2. Calculate alpha: + - If l2 is True: alpha = sqrt(1 - beta^2) + - If l2 is False: alpha = 1 - beta + 3. Combine inputs: output = alpha * x + beta * y + + As the layer index increases, beta decreases, reducing the contribution + of the transformed input y and increasing the contribution of the + original input x. + + Examples: + >>> decay_residual = DecayResidual(1, 64, 64, power=0.5, l2=True) + >>> x = torch.randn(10, 64) + >>> y = torch.randn(10, 64) + >>> output = decay_residual.forward(x, y, transposed=False) + >>> output.shape + torch.Size([10, 64]) + """ beta = self.i_layer ** (-self.power) if self.l2: alpha = (1.0 - beta**2) ** 0.5 diff --git a/espnet2/asr/state_spaces/s4.py b/espnet2/asr/state_spaces/s4.py index b29d7b9cc48..123de939198 100644 --- a/espnet2/asr/state_spaces/s4.py +++ b/espnet2/asr/state_spaces/s4.py @@ -24,10 +24,28 @@ def rank_zero_only(fn: Callable) -> Callable: - """Decorator function from PyTorch Lightning. + """ + Decorator to make a function or method run only on the main process in distributed training. + + This decorator is useful for operations that should only be performed once + across all processes, such as logging or saving checkpoints. + + Args: + fn (Callable): The function to be decorated. + + Returns: + Callable: A wrapped version of the input function that will only execute + when the global rank is 0. - Function that can be used as a decorator - to enable a function/method being called only on global rank 0. + Example: + @rank_zero_only + def log_something(): + print("This will only be printed on the main process") + + Note: + This function assumes that the global rank can be determined from + environment variables. It checks for 'RANK', 'LOCAL_RANK', + 'SLURM_PROCID', and 'JSM_NAMESPACE_RANK' in that order. """ @wraps(fn) @@ -56,7 +74,31 @@ def _get_rank() -> int: def get_logger(name=__name__, level=logging.INFO) -> logging.Logger: - """Initialize multi-GPU-friendly python logger.""" + """ + Initialize a multi-GPU-friendly Python logger. + + This function creates a logger that is aware of distributed training environments. + It decorates all logging methods with the rank_zero_only decorator to ensure + that log messages are only printed once in multi-GPU setups. + + Args: + name (str, optional): The name of the logger. Defaults to the name of the + current module. + level (int, optional): The logging level. Defaults to logging.INFO. + + Returns: + logging.Logger: A configured logger instance with rank-zero-only decorated + logging methods. + + Example: + logger = get_logger("my_module") + logger.info("This message will only be logged once in multi-GPU training") + + Note: + This function modifies the behavior of the standard logging methods + (debug, info, warning, error, exception, fatal, critical) to only execute + on the main process (rank zero) in distributed environments. + """ logger = logging.getLogger(name) logger.setLevel(level) @@ -260,10 +302,33 @@ def _resolve_conj(x): def power(L, A, v=None): - """Compute A^L and the scan sum_i A^i v_i. - - A: (..., N, N) - v: (..., N, L) + """ + Compute A^L and the scan sum_i A^i v_i. + + This function calculates the matrix power A^L and optionally computes + the sum of A^i * v_i for i from 0 to L-1, where v_i is the i-th column of v. + + Args: + L (int): The power to raise A to. + A (torch.Tensor): Square matrix of shape (..., N, N). + v (torch.Tensor, optional): Tensor of shape (..., N, L). If provided, + the function also computes the scan sum. Defaults to None. + + Returns: + torch.Tensor or Tuple[torch.Tensor, torch.Tensor]: + If v is None, returns A^L of shape (..., N, N). + If v is provided, returns a tuple (A^L, sum_i A^i v_i) where + sum_i A^i v_i has shape (..., N). + + Note: + This function uses a divide-and-conquer strategy to compute the + matrix power efficiently. It's particularly useful for large L values. + + Example: + A = torch.randn(3, 3) + v = torch.randn(3, 5) + result = power(5, A, v) + # result is a tuple (A^5, sum of A^i * v_i for i from 0 to 4) """ E = torch.eye(A.shape[-1]).to(A) # , dtype=A.dtype, device=A.device) @@ -309,7 +374,37 @@ def power(L, A, v=None): def transition(measure, N): - """A, B transition matrices for different measures.""" + """ + Compute the A and B transition matrices for different measures. + + This function generates the A and B matrices used in state space models + for various types of measures or basis functions. + + Args: + measure (str): The type of measure to use. Options include: + 'legt' (Legendre (translated)), + 'legs' (Legendre (scaled)), + 'legsd' (Scaled Legendre with diagonal correction), + 'fourier_diag' or 'foud' (Fourier diagonal), + 'fourier' or 'fout' (Fourier). + N (int): The size of the state space (number of basis functions). + + Returns: + Tuple[np.ndarray, np.ndarray]: A tuple containing: + - A (np.ndarray): The transition matrix of shape (N, N). + - B (np.ndarray): The input matrix of shape (N, 1). + + Raises: + NotImplementedError: If an unsupported measure is specified. + + Note: + The matrices are returned as NumPy arrays and may need to be + converted to PyTorch tensors for use in neural network modules. + + Example: + A, B = transition('legt', 64) + # A is a (64, 64) matrix, B is a (64, 1) matrix + """ # Legendre (translated) if measure == "legt": Q = np.arange(N, dtype=np.float64) @@ -375,7 +470,37 @@ def transition(measure, N): def rank_correction(measure, N, rank=1, dtype=torch.float): - """Return low-rank matrix L such that A + L is normal.""" + """ + Return a low-rank matrix L such that A + L is normal. + + This function computes a low-rank correction matrix for different types of + measures used in state space models. The correction ensures that A + L + has the desired normality properties. + + Args: + measure (str): The type of measure. Options include: + 'legs' (Legendre scaled), + 'legt' (Legendre translated), + 'fourier' or 'fout' (Fourier), + 'fourier_diag', 'foud', or 'legsd' (Diagonal corrections). + N (int): The size of the state space. + rank (int, optional): The rank of the correction matrix. Defaults to 1. + dtype (torch.dtype, optional): The data type of the output. Defaults to torch.float. + + Returns: + torch.Tensor: A low-rank correction matrix of shape (rank, N). + + Raises: + NotImplementedError: If an unsupported measure is specified. + + Note: + The returned matrix P is such that PP^* is the desired low-rank + correction. The actual correction is applied as A + PP^*. + + Example: + P = rank_correction('legs', 64, rank=2) + # P is a (2, 64) matrix + """ if measure == "legs": assert rank >= 1 P = torch.sqrt(0.5 + torch.arange(N, dtype=dtype)).unsqueeze(0) # (1 N) @@ -407,11 +532,39 @@ def rank_correction(measure, N, rank=1, dtype=torch.float): def nplr(measure, N, rank=1, dtype=torch.float, diagonalize_precision=True): - """Decompose as Normal Plus Low-Rank (NPLR). - - Return w, p, q, V, B such that - (w - p q^*, B) is unitarily equivalent to the original HiPPO A, B by the matrix V - i.e. A = V[w - p q^*]V^*, B = V B + """ + Decompose the HiPPO matrix into Normal Plus Low-Rank (NPLR) form. + + This function computes the NPLR decomposition of the HiPPO matrix for a given measure. + The decomposition is of the form A = V[w - p q^*]V^*, B = V B, where w is diagonal. + + Args: + measure (str): The type of HiPPO measure (e.g., 'legs', 'legt', 'fourier'). + N (int): The size of the state space. + rank (int, optional): The rank of the low-rank correction. Defaults to 1. + dtype (torch.dtype, optional): The data type for the computations. Defaults to torch.float. + diagonalize_precision (bool, optional): Whether to use higher precision for diagonalization. Defaults to True. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: + - w (torch.Tensor): The diagonal part of the decomposition. + - P (torch.Tensor): The low-rank factor p. + - B (torch.Tensor): The transformed input matrix. + - V (torch.Tensor): The transformation matrix. + + Raises: + NotImplementedError: If the specified measure is not implemented. + + Note: + This function is crucial for the efficient implementation of the S4 model. + The returned matrices satisfy the relation A = V[w - p q^*]V^*, B = V B. + + Example: + w, P, B, V = nplr('legs', 64, rank=2) + # w is a complex vector of length 32 + # P is a (2, 64) matrix + # B is a vector of length 64 + # V is a (64, 64) unitary matrix """ assert dtype == torch.float or torch.double cdtype = torch.cfloat if dtype == torch.float else torch.cdouble @@ -492,6 +645,44 @@ def dplr( diagonal=True, random_B=False, ): + """ + Generate parameters for the Diagonal Plus Low-Rank (DPLR) parameterization of SSMs. + + This function creates the parameters needed for the DPLR version of the S4 model, + which uses a diagonal matrix plus a low-rank correction. + + Args: + scaling (str): The type of scaling to use for the imaginary part ('random', 'real', 'linear', 'inverse', 'inverse2', 'quadratic', 'legs', 'hippo'). + N (int): Size of the state space (half the size due to complex conjugacy). + rank (int, optional): Rank of the low-rank component. Defaults to 1. + H (int, optional): Number of independent SSM copies. Defaults to 1. + dtype (torch.dtype, optional): Data type of the generated tensors. Defaults to torch.float. + real_scale (float, optional): Scaling factor for the real part. Defaults to 1.0. + imag_scale (float, optional): Scaling factor for the imaginary part. Defaults to 1.0. + random_real (bool, optional): Whether to use random initialization for the real part. Defaults to False. + random_imag (bool, optional): Whether to use random initialization for the imaginary part. Defaults to False. + normalize (bool, optional): Whether to normalize the B vector. Defaults to False. + diagonal (bool, optional): Whether to use a diagonal matrix for P. Defaults to True. + random_B (bool, optional): Whether to use random initialization for B. Defaults to False. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: + - w (torch.Tensor): Complex diagonal coefficients of shape (H, N). + - P (torch.Tensor): Low-rank term of shape (rank, H, N). + - B (torch.Tensor): Input projection vector of shape (H, N). + - V (torch.Tensor): Transformation matrix of shape (H, N, N). + + Note: + This function is used to initialize the S4D model, which is a simplification + of the full S4 model using diagonal state matrices. + + Example: + w, P, B, V = dplr('linear', 64, rank=2, H=4) + # w is a complex tensor of shape (4, 32) + # P is a complex tensor of shape (2, 4, 32) + # B is a complex tensor of shape (4, 32) + # V is a complex tensor of shape (4, 64, 64) + """ assert dtype == torch.float or torch.double dtype = torch.cfloat if dtype == torch.float else torch.cdouble @@ -556,11 +747,41 @@ def dplr( def ssm(measure, N, R, H, **ssm_args): - """Dispatcher to create single SSM initialization. - - N: state size - R: rank (for DPLR parameterization) - H: number of independent SSM copies + """ + Create a single State Space Model (SSM) initialization. + + This function serves as a dispatcher to create SSM parameters based on the + specified measure. It supports both NPLR (Normal Plus Low-Rank) and DPLR + (Diagonal Plus Low-Rank) parameterizations. + + Args: + measure (str): The type of measure to use for initialization. Special + values include 'dplr' for Diagonal Plus Low-Rank and 'diag-*' for + various diagonal initializations. + N (int): State size. + R (int): Rank for the low-rank component (used in DPLR parameterization). + H (int): Number of independent SSM copies. + **ssm_args: Additional keyword arguments passed to the specific + initialization functions. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: + - w (torch.Tensor): Diagonal coefficients, shape (H, N). + - P (torch.Tensor): Low-rank term, shape (R, H, N). + - B (torch.Tensor): Input projection, shape (H, N). + - V (torch.Tensor): Transformation matrix, shape (H, N, N). + + Raises: + NotImplementedError: If an unsupported measure is specified. + + Note: + For 'dplr' and 'diag-*' measures, the function calls `dplr()`. + For other measures, it calls `nplr()`. + + Example: + w, P, B, V = ssm('legs', 64, 2, 4) + # Initializes SSM parameters for Legendre measure with + # state size 64, rank 2, and 4 independent copies """ if measure == "dplr": w, P, B, V = dplr(N=N, rank=R, H=H, **ssm_args) @@ -586,6 +807,41 @@ def ssm(measure, N, R, H, **ssm_args): def combination(measures, N, R, S, **ssm_args): + """ + Create a combination of multiple State Space Model (SSM) initializations. + + This function allows for the creation of SSM parameters using multiple measures, + effectively combining different types of SSMs into a single set of parameters. + + Args: + measures (str or list): Either a string specifying a predefined combination + ('hippo', 'diag', 'all') or a list of measure names. + N (int): State size for each SSM. + R (int): Rank for the low-rank component in each SSM. + S (int): Total number of independent trainable SSM copies. + **ssm_args: Additional keyword arguments passed to the ssm() function. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: + - w (torch.Tensor): Combined diagonal coefficients, shape (S, N). + - P (torch.Tensor): Combined low-rank terms, shape (R, S, N). + - B (torch.Tensor): Combined input projections, shape (S, N). + - V (torch.Tensor): Combined transformation matrices, shape (S, N, N). + + Raises: + AssertionError: If S is not divisible by the number of measures. + + Note: + Predefined combinations: + - 'hippo': Combines 'legs' and 'fourier' measures. + - 'diag': Combines 'diag-inv' and 'diag-lin' measures. + - 'all': Combines 'legs', 'fourier', 'diag-inv', and 'diag-lin' measures. + + Example: + w, P, B, V = combination('hippo', 64, 2, 8) + # Creates a combination of Legendre and Fourier SSMs + # with state size 64, rank 2, and 8 total copies (4 of each type) + """ if isinstance(measures, str): measures = combinations[measures] if measures in combinations else [measures] @@ -604,10 +860,42 @@ def combination(measures, N, R, S, **ssm_args): class OptimModule(nn.Module): - """Interface for Module that allows registering buffers/parameters with configurable optimizer hyperparameters. # noqa""" + """ + A module that allows registering buffers/parameters with configurable optimizer hyperparameters. + + This class extends nn.Module to provide a custom registration method for tensors, + allowing them to be registered as either buffers or parameters with specific + learning rates and weight decay settings. + + The main feature is the ability to set per-parameter learning rates, + which can be useful for fine-tuning specific parts of a model or + implementing more complex optimization strategies. + """ def register(self, name, tensor, lr=None): - """Register a tensor with a configurable learning rate and 0 weight decay.""" + """ + Register a tensor as a buffer or parameter with configurable learning rate. + + This method allows for flexible registration of tensors within the module, + with the option to specify a custom learning rate for parameters. + + Args: + name (str): The name under which the tensor will be registered. + tensor (torch.Tensor): The tensor to be registered. + lr (float, optional): The learning rate for the tensor if registered as a parameter. + If None, uses the default learning rate. If 0.0, registers as a buffer. + + Note: + - If lr is 0.0, the tensor is registered as a buffer (non-learnable). + - If lr is not None and not 0.0, the tensor is registered as a parameter + with the specified learning rate and zero weight decay. + - The learning rate is stored in the '_optim' attribute of the parameter, + which can be used by custom optimizers. + + Example: + self.register('my_tensor', torch.randn(10), lr=0.01) + # Registers 'my_tensor' as a parameter with learning rate 0.01 + """ if lr == 0.0: self.register_buffer(name, tensor) else: @@ -620,10 +908,18 @@ def register(self, name, tensor, lr=None): class SSKernelNPLR(OptimModule): - """Stores a representation of and computes the SSKernel function. + """ + Stores a representation of and computes the SSKernel function K_L(A^dt, B^dt, C). - K_L(A^dt, B^dt, C) corresponding to a discretized state space, - where A is Normal + Low Rank (NPLR) + This class implements the Normal Plus Low-Rank (NPLR) parameterization of the + SSM (State Space Model) kernel. It provides methods to compute the kernel function + and its state space realization for the discretized state space model. + + The kernel is represented as K_L(A^dt, B^dt, C) where A is parameterized + as A = w - PP^*, w is complex diagonal, and P is low rank. + + This implementation includes various optimizations and caching mechanisms + to improve computational efficiency, especially for long sequence lengths. """ @torch.no_grad() @@ -795,15 +1091,33 @@ def _w(self): return w def forward(self, state=None, rate=1.0, L=None): - """Forward pass. - - state: (B, H, N) initial state - rate: sampling rate factor - L: target length - - returns: - (C, H, L) convolution kernel (generally C=1) - (B, H, L) output from initial state + """ + Compute the SSKernel function for the given parameters. + + This method calculates the convolution kernel and optionally the output + from an initial state. + + Args: + state (torch.Tensor, optional): Initial state of shape (B, H, N). + If provided, the method also computes the output from this state. + rate (float, optional): Sampling rate factor. Defaults to 1.0. + L (int, optional): Target length of the kernel. If None, uses the + internal length self.L. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - The computed convolution kernel of shape (C, H, L). + - If state is provided, also returns the output from the initial + state of shape (B, H, L). + + Note: + - The method automatically handles length adjustment and caching. + - It supports various discretization methods specified in self.disc. + - The computation leverages FFT for efficiency in long sequence computations. + + Example: + kernel, _ = sskernel.forward(L=1024) + # Computes the kernel for a sequence of length 1024 """ # Initialize C~ # if necessary (done in forward pass so it's on the correct device) @@ -1112,6 +1426,30 @@ def _setup_step(self, mode="dense"): ) def default_state(self, *batch_shape): + """ + Create a default initial state for the SSM. + + This method generates a zero-initialized state tensor with the appropriate + shape and data type for the SSM. + + Args: + *batch_shape: Variable length argument to specify the batch dimensions + of the state tensor. + + Returns: + torch.Tensor: A zero-initialized state tensor of shape (*batch_shape, H, N), + where H is the number of independent SSM copies and N is the state size. + + Note: + - The state tensor is created on the same device as the model parameters. + - The data type of the state matches that of the model's parameters. + - This method is useful for initializing the state when starting a new sequence. + + Example: + state = sskernel.default_state(32, 10) + # Creates a state tensor of shape (32, 10, H, N) for a batch of 32 sequences, + # each with 10 independent dimensions. + """ C = _r2c(self.C) N = C.size(-1) H = C.size(-2) @@ -1153,10 +1491,33 @@ def default_state(self, *batch_shape): return state def step(self, u, state): - """Step one time step as a recurrent model. - - Must have called self._setup_step() - and created state with self.default_state() before calling this + """ + Perform a single step update of the SSM. + + This method applies one step of the state space model update, computing + the new state and output given an input and the current state. + + Args: + u (torch.Tensor): Input tensor of shape (B, H), where B is the batch size + and H is the number of independent SSM copies. + state (torch.Tensor): Current state tensor of shape (B, H, N), where N + is the state size. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - y (torch.Tensor): Output tensor of shape (B, C, H), where C is the + number of output channels. + - next_state (torch.Tensor): Updated state tensor of shape (B, H, N). + + Note: + - This method is intended for use during inference or validation. + - It assumes that self._setup_step() has been called to prepare the + necessary matrices for stepping. + - The computation uses the discretized state space matrices (dA, dB, dC). + + Example: + y, next_state = sskernel.step(u, state) + # Computes the output and next state for a single time step """ if self._step_mode == "linear": new_state = self._step_state_linear(u, state) @@ -1167,7 +1528,21 @@ def step(self, u, state): class SSKernelDiag(OptimModule): - """Version using (complex) diagonal state matrix (S4D).""" + """ + Implements the SSKernel using a diagonal state matrix (S4D). + + This class represents a simplified version of the SSKernel where the state + matrix A is diagonal, leading to more efficient computations. It's part of + the S4D (Simplified State Space Sequence Model) architecture. + + The diagonal parameterization allows for faster forward and backward passes, + making it suitable for longer sequence modeling tasks. This implementation + supports various initialization schemes and discretization methods for the + state space model. + + The kernel is parameterized by diagonal matrices A and B, and a general C matrix, + allowing for efficient computation of the SSM kernel function. + """ def __init__( self, @@ -1246,15 +1621,34 @@ def _A(self): return A def forward(self, L, state=None, rate=1.0, u=None): - """Forward pass. - - state: (B, H, N) initial state - rate: sampling rate factor - L: target length - - returns: - (C, H, L) convolution kernel (generally C=1) - (B, H, L) output from initial state + """ + Compute the SSKernel (S4D) function for the given parameters. + + This method calculates the convolution kernel and optionally the output + from an initial state for the diagonal SSM (S4D) model. + + Args: + L (int): Target length of the kernel. + state (torch.Tensor, optional): Initial state of shape (B, H, N). + If provided, the method also computes the output from this state. + rate (float, optional): Sampling rate factor. Defaults to 1.0. + u (torch.Tensor, optional): Input sequence. Not used in the current implementation. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - The computed convolution kernel of shape (C, H, L). + - If state is provided, also returns the output from the initial + state of shape (B, C, H, L). + + Note: + - The method supports different discretization methods (zoh, bilinear, dss) + specified in self.disc. + - It uses efficient computations leveraging the diagonal structure of A. + - The method handles rate adjustment and supports optional bandlimiting. + + Example: + kernel, _ = sskernel_diag.forward(L=1024, rate=0.5) + # Computes the kernel for a sequence of length 1024 at half the original rate """ dt = torch.exp(self.log_dt) * rate # (H) C = _r2c(self.C) # (C H N) @@ -1344,6 +1738,31 @@ def _setup_step(self): ) # or * dtA / A def default_state(self, *batch_shape): + """ + Create a default initial state for the SSM with diagonal state matrix. + + This method generates a zero-initialized state tensor with the appropriate + shape and data type for the S4D (Simplified State Space Sequence) model. + + Args: + *batch_shape: Variable length argument to specify the batch dimensions + of the state tensor. + + Returns: + torch.Tensor: A zero-initialized state tensor of shape (*batch_shape, H, N), + where H is the number of independent SSM copies and N is the state size. + + Note: + - The state tensor is created on the same device as the model parameters. + - The data type of the state matches that of the model's C parameter. + - This method is useful for initializing the state when starting a new sequence + or batch of sequences. + + Example: + state = sskernel_diag.default_state(32, 10) + # Creates a state tensor of shape (32, 10, H, N) for a batch of 32 sequences, + # each with 10 independent dimensions. + """ C = _r2c(self.C) state = torch.zeros( *batch_shape, self.H, self.N, dtype=C.dtype, device=C.device @@ -1351,6 +1770,36 @@ def default_state(self, *batch_shape): return state def step(self, u, state): + """ + Perform a single step update of the SSM with diagonal state matrix. + + This method applies one step of the state space model update for the S4D model, + computing the new state and output given an input and the current state. + + Args: + u (torch.Tensor): Input tensor of shape (B, H), where B is the batch size + and H is the number of independent SSM copies. + state (torch.Tensor): Current state tensor of shape (B, H, N), where N + is the state size. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - y (torch.Tensor): Output tensor of shape (B, C, H), where C is the + number of output channels. + - next_state (torch.Tensor): Updated state tensor of shape (B, H, N). + + Note: + - This method is optimized for the diagonal structure of the state matrix. + - It assumes that self._setup_step() has been called to prepare the + necessary matrices for stepping. + - The computation uses the discretized state space matrices (dA, dB, dC). + - The output is real-valued, obtained by taking twice the real part of + the complex computation. + + Example: + y, next_state = sskernel_diag.step(u, state) + # Computes the output and next state for a single time step in the S4D model + """ next_state = contract("h n, b h n -> b h n", self.dA, state) + contract( "h n, b h -> b h n", self.dB, u ) @@ -1358,7 +1807,33 @@ def step(self, u, state): return 2 * y.real, next_state def forward_state(self, u, state): - self._setup_step() + """ + Compute the next state of the SSM given an input sequence. + + This method calculates the final state after processing an entire input sequence + using the S4D (Simplified State Space Sequence) model with a diagonal state matrix. + + Args: + u (torch.Tensor): Input sequence tensor of shape (B, H, L), where B is the + batch size, H is the number of independent SSM copies, + and L is the sequence length. + state (torch.Tensor): Initial state tensor of shape (B, H, N), where N + is the state size. + + Returns: + torch.Tensor: The final state after processing the entire input sequence, + with shape (B, H, N). + + Note: + - This method is more efficient than repeatedly calling step() for each + time step, as it computes the state transition for the entire sequence at once. + - It uses the log_vandermonde_transpose function for efficient computation. + - The method automatically sets up the step matrices if not already done. + + Example: + final_state = sskernel_diag.forward_state(input_sequence, initial_state) + # Computes the final state after processing the entire input_sequence + """ AL = self.dA ** u.size(-1) u = u.flip(-1).to(self.dA).contiguous() # (B H L) v = log_vandermonde_transpose(u, self.dB, self.dA.log(), u.size(-1)) @@ -1367,13 +1842,18 @@ def forward_state(self, u, state): class SSKernel(nn.Module): - """Wrapper around SSKernel parameterizations. + """ + Wrapper class for different State Space Kernel parameterizations. + + This class serves as a unified interface for various implementations of + State Space Kernels, including NPLR (Normal Plus Low-Rank) and Diagonal + parameterizations. It provides a common API for initializing and using + different SSM (State Space Model) kernels, making it easier to experiment + with and switch between different kernel types. - The SSKernel is expected to support the interface - forward() - default_state() - _setup_step() - step() + The SSKernel supports various initialization schemes, measures, and + configuration options, allowing for flexible and powerful sequence modeling. + It can be used as a core component in S4 (Structured State Space Sequence) models. """ def __init__( @@ -1510,18 +1990,66 @@ def __init__( raise NotImplementedError(f"mode={mode} is not valid") def forward(self, state=None, L=None, rate=None): + """ + Compute the SSKernel function for the given parameters. + + This method is a wrapper around the underlying kernel's forward method, + providing a unified interface for different kernel implementations. + + Args: + state (torch.Tensor, optional): Initial state. Shape depends on the + specific kernel implementation. + L (int, optional): Target length of the kernel. If None, uses the + default length specified in the kernel. + rate (float, optional): Sampling rate factor. Defaults to None. + + Returns: + The return type and shape depend on the specific kernel implementation, + but typically includes: + - The computed convolution kernel + - Optionally, the output from the initial state if provided + + Note: + This method directly calls the forward method of the underlying + kernel (either SSKernelNPLR or SSKernelDiag), passing through all + provided arguments. + + Example: + kernel, state_output = sskernel(state=initial_state, L=1024, rate=0.5) + # Computes the kernel and state output for a sequence of length 1024 + # at half the original rate + """ return self.kernel(state=state, L=L, rate=rate) @torch.no_grad() def forward_state(self, u, state): - """Forward the state through a sequence. - - i.e. computes the state after passing chunk through SSM - - state: (B, H, N) - u: (B, H, L) - - Returns: (B, H, N) + """ + Compute the next state of the SSM given an input sequence. + + This method forwards the state through a sequence using the underlying + kernel's state transition logic. + + Args: + u (torch.Tensor): Input sequence tensor. The shape should be + compatible with the underlying kernel, typically (B, H, L), + where B is batch size, H is the number of independent SSM copies, + and L is the sequence length. + state (torch.Tensor): Initial state tensor. The shape should match + the state representation of the underlying kernel. + + Returns: + torch.Tensor: The final state after processing the entire input sequence. + The shape will match the input state shape. + + Note: + - This method is particularly useful for tracking the state evolution + over a sequence without computing the full convolution output. + - If the underlying kernel doesn't have a `forward_state` method, + this function falls back to a manual state update using `_setup_state`. + + Example: + final_state = sskernel.forward_state(input_sequence, initial_state) + # Computes the final state after processing the entire input_sequence """ if hasattr(self.kernel, "forward_state"): return self.kernel.forward_state(u, state) @@ -1553,14 +2081,91 @@ def _setup_step(self, **kwargs): self.kernel._setup_step(**kwargs) def step(self, u, state, **kwargs): + """ + Perform a single step update of the SSM. + + This method applies one step of the state space model update, computing + the new state and output given an input and the current state. It delegates + the computation to the underlying kernel's step method. + + Args: + u (torch.Tensor): Input tensor for the current step. The shape should be + compatible with the underlying kernel, typically (B, H) where B is + the batch size and H is the number of independent SSM copies. + state (torch.Tensor): Current state tensor. The shape should match the + state representation of the underlying kernel. + **kwargs: Additional keyword arguments that will be passed to the + underlying kernel's step method. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - y (torch.Tensor): Output tensor for the current step. + - next_state (torch.Tensor): Updated state tensor. + + Note: + - This method is intended for use during inference or validation, allowing + for step-by-step processing of a sequence. + - It assumes that the underlying kernel has been properly initialized and + its `_setup_step` method has been called if necessary. + + Example: + output, new_state = sskernel.step(input, current_state) + # Computes the output and next state for a single time step + """ y, state = self.kernel.step(u, state, **kwargs) return y, state def default_state(self, *args, **kwargs): + """ + Create a default initial state for the SSM. + + This method generates a default state tensor appropriate for the underlying + kernel implementation. It serves as a convenience wrapper around the + kernel's own default_state method. + + Args: + *args: Variable length argument list that will be passed to the + underlying kernel's default_state method. + **kwargs: Arbitrary keyword arguments that will be passed to the + underlying kernel's default_state method. + + Returns: + torch.Tensor: A default state tensor. The shape and content of this + tensor depend on the specific implementation of the + underlying kernel. + + Note: + - The returned state is typically a zero-initialized tensor with + appropriate dimensions for the SSM. + - This method is useful for initializing the state when starting + a new sequence or when no prior state information is available. + + Example: + initial_state = sskernel.default_state(32, 10) + # Creates a default state tensor for a batch of 32 sequences, + # potentially with 10 independent dimensions (depending on the kernel) + """ return self.kernel.default_state(*args, **kwargs) class S4(nn.Module): + """ + Structured State Space Sequence (S4) model. + + This class implements the S4 model, which combines a State Space Model (SSM) + with position-wise feedforward layers. S4 is designed for efficient and + effective modeling of long sequences. + + The S4 module includes: + - An SSM kernel for sequence modeling + - Optional gating and bottleneck mechanisms + - Position-wise feedforward layers with various activation functions + - Support for bidirectional processing + + The model can be configured with different SSM kernels (e.g., NPLR or diagonal) + and allows for extensive customization of its components and hyperparameters. + """ + def __init__( self, d_model, @@ -1691,12 +2296,35 @@ def __init__( ) def forward(self, u, state=None, rate=1.0, lengths=None, **kwargs): - """Forward pass. - - u: (B H L) if self.transposed else (B L H) - state: (H N) never needed unless you know what you're doing - - Returns: same shape as u + """ + Forward pass of the S4 model. + + This method applies the S4 transformation to the input sequence, including + the SSM convolution and feedforward layers. + + Args: + u (torch.Tensor): Input tensor of shape (B, H, L) if self.transposed else (B, L, H), + where B is batch size, H is hidden size, and L is sequence length. + state (torch.Tensor, optional): Initial state for the SSM. Shape depends on the SSM configuration. + rate (float, optional): Sampling rate factor for the SSM. Defaults to 1.0. + lengths (torch.Tensor or int, optional): Actual lengths of sequences in the batch. + Used for masking padded elements. + **kwargs: Additional keyword arguments passed to the SSM kernel. + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - Output tensor of the same shape as the input. + - Next state of the SSM if state argument was provided, else None. + + Note: + - The method handles different axis orderings based on self.transposed. + - It applies sequence masking if lengths are provided. + - The SSM convolution is combined with skip connections and feedforward layers. + - Supports optional gating and bottleneck mechanisms if configured. + + Example: + output, next_state = s4_model(input_sequence, state=initial_state, rate=0.5) + # Processes the input_sequence and returns the output along with the next state """ if not self.transposed: u = u.transpose(-1, -2) @@ -1779,16 +2407,58 @@ def forward(self, u, state=None, rate=1.0, lengths=None, **kwargs): return y, next_state def setup_step(self, **kwargs): - self.kernel._setup_step(**kwargs) + """ + Set up the S4 model for step-by-step inference. + + This method prepares the S4 model for sequential processing, typically used + during inference or generation tasks. It initializes the underlying SSM kernel + for step-wise computation. + + Args: + **kwargs: Arbitrary keyword arguments that will be passed to the + SSM kernel's _setup_step method. + + Note: + - This method should be called before using the step() method for + sequential processing. + - It configures the internal state of the SSM kernel for efficient + step-by-step computation. + - The exact behavior depends on the specific SSM kernel implementation. + + Example: + s4_model.setup_step() + # Prepares the model for subsequent calls to s4_model.step() + """ def step(self, u, state, **kwargs): - """Step one time step as a recurrent model. - - Intended to be used during validation. - - u: (B H) - state: (B H N) - Returns: output (B H), state (B H N) + """ + Perform a single step update of the S4 model. + + This method processes a single time step input, updating the model's state + and producing an output. It's designed for use in sequential processing + scenarios, typically during inference or generation. + + Args: + u (torch.Tensor): Input tensor for the current step. Shape should be (B, H) + where B is the batch size and H is the hidden size. + state (torch.Tensor): Current state of the model. The shape depends on + the specific SSM kernel configuration. + **kwargs: Additional keyword arguments passed to the SSM kernel's step method. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + - y (torch.Tensor): Output tensor for the current step, shape (B, H). + - next_state (torch.Tensor): Updated state tensor for use in the next step. + + Note: + - This method assumes that setup_step() has been called beforehand. + - It applies the SSM kernel step, followed by activation and the output linear layer. + - The method is not intended for training, but for inference or generation. + - If configured, it applies gating mechanism to the output. + + Example: + output, new_state = s4_model.step(input_step, current_state) + # Processes a single time step and returns the output and updated state """ assert not self.training y, next_state = self.kernel.step(u, state) # (B C H) @@ -1802,10 +2472,57 @@ def step(self, u, state, **kwargs): return y, next_state def default_state(self, *batch_shape, device=None): + """ + Create a default initial state for the S4 model. + + This method generates a default state tensor appropriate for the S4 model's + SSM kernel. It's typically used to initialize the state at the beginning of + a sequence processing task. + + Args: + *batch_shape: Variable length argument list specifying the batch dimensions + of the state tensor. + device (torch.device, optional): The device on which to create the state tensor. + If None, uses the device of the model's parameters. + + Returns: + torch.Tensor: A default state tensor. The shape and content of this tensor + depend on the specific implementation of the underlying SSM kernel. + + Note: + - The returned state is typically a zero-initialized tensor with dimensions + that match the SSM kernel's requirements. + - This method is useful when starting sequence processing without any prior state, + or when resetting the model's state. + - The device of the returned tensor will match the model's parameters if not specified. + + Example: + initial_state = s4_model.default_state(32) + # Creates a default state tensor for a batch of 32 sequences + """ # kernel is not a SequenceModule so it doesn't need to adhere to same interface # the kernel will know the device of its own parameters return self.kernel.default_state(*batch_shape) @property def d_output(self): + """ + Get the output dimension of the S4 model. + + This property returns the dimension of the output produced by the S4 model + for each time step. + + Returns: + int: The output dimension, which is equal to the model's hidden dimension (d_model). + + Note: + - This is typically used to determine the size of the model's output + for downstream tasks or for connecting to other layers. + - The output dimension is the same as the input dimension (d_model) in the + standard S4 configuration, maintaining the sequence's channel size. + + Example: + output_size = s4_model.d_output + # Retrieves the output dimension of the S4 model + """ return self.d_model diff --git a/espnet2/asr/state_spaces/utils.py b/espnet2/asr/state_spaces/utils.py index 7234cf07888..af8d9a1d2d9 100644 --- a/espnet2/asr/state_spaces/utils.py +++ b/espnet2/asr/state_spaces/utils.py @@ -9,17 +9,85 @@ def is_list(x): + """ + Check if the input is a list-like sequence (excluding strings). + + This function determines whether the input is a sequence (like a list or tuple) + but not a string. + + Args: + x: The object to be checked. + + Returns: + bool: True if the input is a list-like sequence, False otherwise. + + Examples: + >>> is_list([1, 2, 3]) + True + >>> is_list((1, 2, 3)) + True + >>> is_list("abc") + False + >>> is_list(123) + False + """ return isinstance(x, Sequence) and not isinstance(x, str) def is_dict(x): + """ + Check if the input is a dictionary-like object. + + This function determines whether the input is an instance of a Mapping, + which includes dictionaries and other mapping types. + + Args: + x: The object to be checked. + + Returns: + bool: True if the input is a dictionary-like object, False otherwise. + + Examples: + >>> is_dict({'a': 1, 'b': 2}) + True + >>> is_dict(dict(a=1, b=2)) + True + >>> is_dict([1, 2, 3]) + False + >>> is_dict("abc") + False + """ return isinstance(x, Mapping) def to_dict(x, recursive=True): - """Convert Sequence or Mapping object to dict. + """ + Convert a Sequence or Mapping object to a dictionary. - lists get converted to {0: x[0], 1: x[1], ...} + This function converts list-like objects to dictionaries with integer keys, + and recursively converts nested structures if specified. + + Args: + x: The object to be converted to a dictionary. + recursive (bool, optional): If True, recursively convert nested structures. + Defaults to True. + + Returns: + dict: The converted dictionary. + + Examples: + >>> to_dict([1, 2, 3]) + {0: 1, 1: 2, 2: 3} + >>> to_dict({'a': [1, 2], 'b': {'c': 3}}, recursive=True) + {'a': {0: 1, 1: 2}, 'b': {'c': 3}} + >>> to_dict({'a': [1, 2], 'b': {'c': 3}}, recursive=False) + {'a': [1, 2], 'b': {'c': 3}} + >>> to_dict(5) + 5 + + Note: + - Lists are converted to dictionaries with integer keys. + - If the input is neither a Sequence nor a Mapping, it is returned unchanged. """ if is_list(x): x = {i: v for i, v in enumerate(x)} @@ -33,11 +101,38 @@ def to_dict(x, recursive=True): def to_list(x, recursive=False): - """Convert an object to list. + """ + Convert an object to a list. + + This function converts Sequence objects (except strings) to lists and handles + recursive conversion of nested structures if specified. + + Args: + x: The object to be converted to a list. + recursive (bool, optional): If True, recursively convert nested structures. + Defaults to False. - If Sequence (e.g. list, tuple, Listconfig): just return it + Returns: + list: The converted list. - Special case: If non-recursive and not a list, wrap in list + Examples: + >>> to_list((1, 2, 3)) + [1, 2, 3] + >>> to_list([1, 2, 3]) + [1, 2, 3] + >>> to_list("abc") + ["abc"] + >>> to_list(5) + [5] + >>> to_list([1, [2, 3], 4], recursive=True) + [1, [2, 3], 4] + >>> to_list([1, [2, 3], 4], recursive=False) + [1, [2, 3], 4] + + Note: + - If the input is not a Sequence and recursive is False, it will be wrapped in a list. + - If recursive is True and the input is not a Sequence, it will be returned unchanged. + - Strings are treated as non-Sequence objects and will be wrapped in a list if recursive is False. """ if is_list(x): if recursive: @@ -52,6 +147,38 @@ def to_list(x, recursive=False): def extract_attrs_from_obj(obj, *attrs): + """ + Extract specified attributes from an object. + + This function retrieves the values of specified attributes from the given object. + If the object is None, it returns an empty list. + + Args: + obj: The object from which to extract attributes. + *attrs: Variable length argument list of attribute names to extract. + + Returns: + list: A list containing the values of the specified attributes. + If an attribute doesn't exist, None is used as its value. + + Raises: + AssertionError: If obj is None and attrs is not empty. + + Examples: + >>> class Example: + ... def __init__(self): + ... self.a = 1 + ... self.b = "test" + >>> obj = Example() + >>> extract_attrs_from_obj(obj, "a", "b", "c") + [1, "test", None] + >>> extract_attrs_from_obj(None) + [] + + Note: + - If obj is None, the function expects no attributes to be specified. + - For non-existent attributes, None is used as the value in the returned list. + """ if obj is None: assert len(attrs) == 0 return [] @@ -59,15 +186,43 @@ def extract_attrs_from_obj(obj, *attrs): def instantiate(registry, config, *args, partial=False, wrap=None, **kwargs): - """Instantiate registered module. + """ + Instantiate a registered module based on the provided configuration. + + This function creates an instance of a registered module using the provided + configuration and additional arguments. It supports various instantiation + scenarios and can optionally wrap the target class. + + Args: + registry (dict): A dictionary mapping names to functions or target paths. + config (dict or str): Configuration for instantiation. If a string, it's + treated as the key for the registry. If a dict, it should contain a + '_name_' key indicating which element of the registry to use. + *args: Additional positional arguments to pass to the constructor. + partial (bool, optional): If True, returns a partial function instead of + an instantiated object. Defaults to False. + wrap (callable, optional): A function to wrap the target class. Defaults to None. + **kwargs: Additional keyword arguments to override the config and pass + to the target constructor. + + Returns: + object or functools.partial: The instantiated object or a partial function, + depending on the 'partial' parameter. - registry: Dictionary mapping names to functions or target paths - (e.g. {'model': 'models.SequenceModel'}) - config: Dictionary with a '_name_' key indicating which element of the registry - to grab, and kwargs to be passed into the target constructor - wrap: wrap the target class (e.g. ema optimizer or tasks.wrap) - *args, **kwargs: additional arguments - to override the config to pass into the target constructor + Raises: + NotImplementedError: If the instantiate target is neither a string nor callable. + + Examples: + >>> registry = {'model': 'models.SequenceModel'} + >>> config = {'_name_': 'model', 'hidden_size': 128} + >>> model = instantiate(registry, config) + >>> partial_model = instantiate(registry, config, partial=True) + >>> wrapped_model = instantiate(registry, config, wrap=some_wrapper_function) + + Note: + - The function supports both string-based and callable-based registry entries. + - If 'config' is a string, it's used as the key for the registry. + - The '_name_' key is restored in the config after instantiation. """ # Case 1: no config if config is None: @@ -106,12 +261,74 @@ def instantiate(registry, config, *args, partial=False, wrap=None, **kwargs): def get_class(registry, _name_): - breakpoint() + """ + Retrieve a class from the registry based on the provided name. + + This function uses Hydra's get_class utility to fetch the class specified + by the _name_ parameter from the given registry. + + Args: + registry (dict): A dictionary mapping names to class paths. + _name_ (str): The name of the class to retrieve from the registry. + + Returns: + type: The class object corresponding to the specified name. + + Raises: + Any exceptions raised by hydra.utils.get_class. + + Examples: + >>> registry = {'MyClass': 'path.to.MyClass'} + >>> MyClass = get_class(registry, 'MyClass') + >>> instance = MyClass() + + Note: + - This function includes a breakpoint() call, which will pause execution + when the function is called. This is likely for debugging purposes and + should be removed in production code. + - The function relies on Hydra's get_class utility, which dynamically + imports and returns the specified class. + """ return hydra.utils.get_class(path=registry[_name_]) def omegaconf_filter_keys(d, fn=None): - """Only keep keys where fn(key) is True. Support nested DictConfig.""" + """ + Filter keys in an OmegaConf structure based on a given function. + + This function traverses through a nested OmegaConf structure (DictConfig or ListConfig) + and filters keys based on the provided function. It supports recursive filtering + for nested structures. + + Args: + d (Union[ListConfig, DictConfig, Any]): The OmegaConf structure to filter. + fn (Callable[[str], bool], optional): A function that takes a key as input + and returns True if the key should be kept, False otherwise. + If None, all keys are kept. Defaults to None. + + Returns: + Union[ListConfig, DictConfig, Any]: A new OmegaConf structure with filtered keys. + + Examples: + >>> from omegaconf import DictConfig, ListConfig + >>> config = DictConfig({'a': 1, 'b': ListConfig([1, 2]), 'c': DictConfig({'d': 3, 'e': 4})}) + >>> filtered = omegaconf_filter_keys(config, lambda k: k != 'b') + >>> print(filtered) + {'a': 1, 'c': {'d': 3, 'e': 4}} + + >>> def filter_func(key): + ... return not key.startswith('_') + >>> config = DictConfig({'a': 1, '_b': 2, 'c': DictConfig({'d': 3, '_e': 4})}) + >>> filtered = omegaconf_filter_keys(config, filter_func) + >>> print(filtered) + {'a': 1, 'c': {'d': 3}} + + Note: + - If no filter function is provided, all keys are kept. + - The function preserves the OmegaConf structure (DictConfig for dictionaries, + ListConfig for lists) in the returned object. + - For non-dict and non-list inputs, the original input is returned unchanged. + """ if fn is None: def fn(_): diff --git a/espnet2/asr/transducer/beam_search_transducer.py b/espnet2/asr/transducer/beam_search_transducer.py index 4d5aa99cfed..b0e269dc551 100644 --- a/espnet2/asr/transducer/beam_search_transducer.py +++ b/espnet2/asr/transducer/beam_search_transducer.py @@ -19,17 +19,17 @@ @dataclass -class Hypothesis: - """Default hypothesis definition for Transducer search algorithms.""" - - score: float - yseq: List[int] - dec_state: Union[ - Tuple[torch.Tensor, Optional[torch.Tensor]], - List[Optional[torch.Tensor]], - torch.Tensor, - ] - lm_state: Union[Dict[str, Any], List[Any]] = None +# class Hypothesis: +# """Default hypothesis definition for Transducer search algorithms.""" + +# score: float +# yseq: List[int] +# dec_state: Union[ +# Tuple[torch.Tensor, Optional[torch.Tensor]], +# List[Optional[torch.Tensor]], +# torch.Tensor, +# ] +# lm_state: Union[Dict[str, Any], List[Any]] = None @dataclass diff --git a/espnet2/asr/transducer/error_calculator.py b/espnet2/asr/transducer/error_calculator.py index 4ddf9cc9b7b..e35ad2aab40 100644 --- a/espnet2/asr/transducer/error_calculator.py +++ b/espnet2/asr/transducer/error_calculator.py @@ -9,16 +9,42 @@ class ErrorCalculatorTransducer(object): - """Calculate CER and WER for transducer models. + """ + Error Calculator for Transducer-based models in speech recognition. - Args: - decoder: Decoder module. - token_list: List of tokens. - sym_space: Space symbol. - sym_blank: Blank symbol. - report_cer: Whether to compute CER. - report_wer: Whether to compute WER. + This class calculates Character Error Rate (CER) and Word Error Rate (WER) for + transducer models used in automatic speech recognition tasks. + Attributes: + beam_search (BeamSearchTransducer): Beam search decoder for transducer models. + decoder (AbsDecoder): Decoder module. + token_list (List[int]): List of token IDs. + space (str): Space symbol. + blank (str): Blank symbol. + report_cer (bool): Flag to compute CER. + report_wer (bool): Flag to compute WER. + + Args: + decoder (AbsDecoder): Decoder module. + joint_network (torch.nn.Module): Joint network module. + token_list (List[int]): List of token IDs. + sym_space (str): Space symbol. + sym_blank (str): Blank symbol. + report_cer (bool, optional): Whether to compute CER. Defaults to False. + report_wer (bool, optional): Whether to compute WER. Defaults to False. + + Example: + >>> decoder = TransducerDecoder(...) + >>> joint_network = JointNetwork(...) + >>> token_list = ["", "a", "b", "c", ...] + >>> error_calc = ErrorCalculatorTransducer( + ... decoder, joint_network, token_list, sym_space="", sym_blank="" + ... ) + >>> cer, wer = error_calc(encoder_out, target) + + Note: + This class uses beam search decoding to generate predictions and then + calculates the error rates by comparing them with the target sequences. """ def __init__( @@ -52,16 +78,38 @@ def __init__( self.report_wer = report_wer def __call__(self, encoder_out: torch.Tensor, target: torch.Tensor): - """Calculate sentence-level WER/CER score for Transducer model. + """ + Calculate sentence-level WER/CER score for Transducer model. + + This method performs beam search decoding on the encoder output and calculates + the Character Error Rate (CER) and Word Error Rate (WER) by comparing the + decoded sequences with the target sequences. Args: - encoder_out: Encoder output sequences. (B, T, D_enc) - target: Target label ID sequences. (B, L) + encoder_out (torch.Tensor): Encoder output sequences. Shape: (B, T, D_enc), + where B is the batch size, T is the sequence length, and D_enc is the + encoder output dimension. + target (torch.Tensor): Target label ID sequences. Shape: (B, L), where B is + the batch size and L is the target sequence length. Returns: - : Sentence-level CER score. - : Sentence-level WER score. - + tuple: A tuple containing: + - cer (float or None): Sentence-level Character Error Rate if report_cer + is True, else None. + - wer (float or None): Sentence-level Word Error Rate if report_wer is + True, else None. + + Example: + >>> encoder_out = torch.randn(2, 100, 256) # Batch size 2, seq length 100 + >>> target = torch.randint(0, 1000, (2, 50)) # Batch size 2, target length 50 + >>> error_calc = ErrorCalculatorTransducer(...) + >>> cer, wer = error_calc(encoder_out, target) + >>> print(f"CER: {cer}, WER: {wer}") + + Note: + This method uses the beam search algorithm to decode the encoder output. + The resulting predictions are then converted to character sequences for + error rate calculation. """ cer, wer = None, None @@ -89,16 +137,38 @@ def __call__(self, encoder_out: torch.Tensor, target: torch.Tensor): def convert_to_char( self, pred: torch.Tensor, target: torch.Tensor ) -> Tuple[List, List]: - """Convert label ID sequences to character sequences. + """ + Convert label ID sequences to character sequences. + + This method transforms the predicted and target label ID sequences into + character sequences by mapping each ID to its corresponding token. It also + handles the replacement of space and blank symbols. Args: - pred: Prediction label ID sequences. (B, U) - target: Target label ID sequences. (B, L) + pred (torch.Tensor): Prediction label ID sequences. Shape: (B, U), where B + is the batch size and U is the length of the predicted sequence. + target (torch.Tensor): Target label ID sequences. Shape: (B, L), where B is + the batch size and L is the length of the target sequence. Returns: - char_pred: Prediction character sequences. (B, ?) - char_target: Target character sequences. (B, ?) - + tuple: A tuple containing two lists: + - char_pred (List[str]): List of predicted character sequences, one for + each sample in the batch. + - char_target (List[str]): List of target character sequences, one for + each sample in the batch. + + Example: + >>> pred = torch.tensor([[1, 2, 3], [4, 5, 6]]) + >>> target = torch.tensor([[1, 2, 3], [4, 5, 6]]) + >>> error_calc = ErrorCalculatorTransducer(...) + >>> char_pred, char_target = error_calc.convert_to_char(pred, target) + >>> print(f"Predictions: {char_pred}") + >>> print(f"Targets: {char_target}") + + Note: + This method replaces the space symbol with an actual space character and + removes any occurrence of the blank symbol in both predicted and target + sequences. """ char_pred, char_target = [], [] @@ -120,15 +190,35 @@ def convert_to_char( def calculate_cer( self, char_pred: torch.Tensor, char_target: torch.Tensor ) -> float: - """Calculate sentence-level CER score. + """ + Calculate sentence-level Character Error Rate (CER) score. + + This method computes the CER by comparing the predicted character sequences + with the target character sequences. It uses the Levenshtein distance + (edit distance) to measure the difference between sequences. Args: - char_pred: Prediction character sequences. (B, ?) - char_target: Target character sequences. (B, ?) + char_pred (List[str]): List of predicted character sequences, one for + each sample in the batch. + char_target (List[str]): List of target character sequences, one for + each sample in the batch. Returns: - : Average sentence-level CER score. - + float: Average sentence-level CER score across the batch. + + Example: + >>> char_pred = ["hello wrld", "how r u"] + >>> char_target = ["hello world", "how are you"] + >>> error_calc = ErrorCalculatorTransducer(...) + >>> cer = error_calc.calculate_cer(char_pred, char_target) + >>> print(f"Character Error Rate: {cer}") + + Note: + - This method removes all space characters from both predicted and target + sequences before calculating the edit distance. + - The CER is calculated as the sum of edit distances divided by the sum of + target sequence lengths. + - This method requires the 'editdistance' package to be installed. """ import editdistance @@ -146,15 +236,35 @@ def calculate_cer( def calculate_wer( self, char_pred: torch.Tensor, char_target: torch.Tensor ) -> float: - """Calculate sentence-level WER score. + """ + Calculate sentence-level Word Error Rate (WER) score. + + This method computes the WER by comparing the predicted word sequences + with the target word sequences. It uses the Levenshtein distance + (edit distance) to measure the difference between word sequences. Args: - char_pred: Prediction character sequences. (B, ?) - char_target: Target character sequences. (B, ?) + char_pred (List[str]): List of predicted character sequences, one for + each sample in the batch. + char_target (List[str]): List of target character sequences, one for + each sample in the batch. Returns: - : Average sentence-level WER score - + float: Average sentence-level WER score across the batch. + + Example: + >>> char_pred = ["hello world how are you", "this is a test"] + >>> char_target = ["hello world how are you doing", "this is only a test"] + >>> error_calc = ErrorCalculatorTransducer(...) + >>> wer = error_calc.calculate_wer(char_pred, char_target) + >>> print(f"Word Error Rate: {wer}") + + Note: + - This method splits each character sequence into words before calculating + the edit distance. + - The WER is calculated as the sum of edit distances divided by the sum of + target word sequence lengths. + - This method requires the 'editdistance' package to be installed. """ import editdistance diff --git a/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py b/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py index c007fa5536d..aabf70f76b1 100644 --- a/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py +++ b/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py @@ -48,25 +48,44 @@ def rnnt_loss_cpu( clamp: float, num_threads: int, ): - """Wrapper method for accessing CPU RNNT loss. + """ + Computes the RNN-T loss on CPU. - CPU implementation ported from [HawkAaron/warp-transducer] - (https://github.com/HawkAaron/warp-transducer). + This function is a wrapper for the CPU implementation of the RNN-T loss, + ported from the warp-transducer project. Args: acts: Activation tensor of shape [B, T, U, V+1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. - label_lengths: Lengths of the target sequence as a vector of ints [B]. - costs: Zero vector of length [B] in which costs will be set. - grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. + input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. + label_lengths: Lengths of the target sequences as a vector of ints [B]. + costs: Zero vector of length [B] where the computed costs will be stored. + grads: Zero tensor of shape [B, T, U, V+1] where the computed gradients will be stored. blank_label: Index of the blank token in the vocabulary. - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. - clamp: Float value. When set to value >= 0.0, will clamp the - gradient to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. + fastemit_lambda: Float scaling factor for FastEmit regularization. + clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. + + Returns: + bool: True if the computation was successful. + + Raises: + RuntimeError: If there's an error in calculating the forward scores or workspace memory. + + Note: + B: batch size, T: input sequence length, U: output sequence length, V: vocabulary size + + Example: + >>> acts = torch.randn(2, 10, 5, 21) + >>> labels = torch.randint(0, 20, (2, 5)) + >>> input_lengths = torch.tensor([10, 8]) + >>> label_lengths = torch.tensor([5, 4]) + >>> costs = torch.zeros(2) + >>> grads = torch.zeros_like(acts) + >>> success = rnnt_loss_cpu(acts, labels, input_lengths, label_lengths, costs, grads, + ... blank_label=0, fastemit_lambda=0.0, clamp=0.0, num_threads=4) + >>> print(success) + True """ # aliases @@ -156,25 +175,44 @@ def rnnt_loss_gpu( clamp: float, num_threads: int, ): - """Wrapper method for accessing GPU RNNT loss. + """ + Computes the RNN-T loss on GPU. - CUDA implementation ported from [HawkAaron/warp-transducer] - (https://github.com/HawkAaron/warp-transducer). + This function is a wrapper for the CUDA implementation of the RNN-T loss, + ported from the warp-transducer project. Args: acts: Activation tensor of shape [B, T, U, V+1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. - label_lengths: Lengths of the target sequence as a vector of ints [B]. - costs: Zero vector of length [B] in which costs will be set. - grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. + input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. + label_lengths: Lengths of the target sequences as a vector of ints [B]. + costs: Zero vector of length [B] where the computed costs will be stored. + grads: Zero tensor of shape [B, T, U, V+1] where the computed gradients will be stored. blank_label: Index of the blank token in the vocabulary. - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. - clamp: Float value. When set to value >= 0.0, will clamp the - gradient to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. + fastemit_lambda: Float scaling factor for FastEmit regularization. + clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. + + Returns: + bool: True if the computation was successful. + + Raises: + RuntimeError: If there's an error in calculating the forward scores or workspace memory. + + Note: + B: batch size, T: input sequence length, U: output sequence length, V: vocabulary size + + Example: + >>> acts = torch.randn(2, 10, 5, 21, device='cuda') + >>> labels = torch.randint(0, 20, (2, 5), device='cuda') + >>> input_lengths = torch.tensor([10, 8], device='cuda') + >>> label_lengths = torch.tensor([5, 4], device='cuda') + >>> costs = torch.zeros(2, device='cuda') + >>> grads = torch.zeros_like(acts) + >>> success = rnnt_loss_gpu(acts, labels, input_lengths, label_lengths, costs, grads, + ... blank_label=0, fastemit_lambda=0.0, clamp=0.0, num_threads=4) + >>> print(success) + True """ minibatch_size = acts.shape[0] @@ -270,35 +308,50 @@ def multiblank_rnnt_loss_gpu( num_threads: int, sigma: float, ): - """Wrapper method for accessing GPU Multi-blank RNNT loss + """ + Computes the Multi-blank RNN-T loss on GPU. + This function is a wrapper for the CUDA implementation of the Multi-blank RNN-T loss, + as described in the paper "Multi-blank Transducers for Speech Recognition" (https://arxiv.org/pdf/2211.03541.pdf). - CUDA implementation ported from [HawkAaron/warp-transducer] - (https://github.com/HawkAaron/warp-transducer). - Args: acts: Activation tensor of shape [B, T, U, V + num_big_blanks + 1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. - label_lengths: Lengths of the target sequence as a vector of ints [B]. - costs: Zero vector of length [B] in which costs will be set. - grads: Zero tensor of shape [B, T, U, V + num_big_blanks + 1] - where the gradient will be set. + input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. + label_lengths: Lengths of the target sequences as a vector of ints [B]. + costs: Zero vector of length [B] where the computed costs will be stored. + grads: Zero tensor of shape [B, T, U, V + num_big_blanks + 1] where the computed gradients will be stored. blank_label: Index of the standard blank token in the vocabulary. - big_blank_durations: A list of supported durations for big blank symbols - in the model, e.g. [2, 4, 8]. Note we only include durations for ``big - blanks'' here and it should not include 1 for the standard blank. - Those big blanks have vocabulary indices after the standard blank index. - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. - clamp: Float value. When set to value >= 0.0, will clamp the - gradient to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. - sigma: logit-undernormalization weight used in the multi-blank model. Refer to - the multi-blank paper https://arxiv.org/pdf/2211.03541 - for detailed explanations. + big_blank_durations: List of supported durations for big blank symbols, e.g. [2, 4, 8]. + fastemit_lambda: Float scaling factor for FastEmit regularization. + clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. + sigma: Logit-undernormalization weight used in the multi-blank model. + + Returns: + bool: True if the computation was successful. + + Raises: + RuntimeError: If there's an error in calculating the forward scores or workspace memory. + + Note: + B: batch size, T: input sequence length, U: output sequence length, + V: vocabulary size, num_big_blanks: number of big blank symbols + + Example: + >>> acts = torch.randn(2, 10, 5, 24, device='cuda') # Assuming 3 big blanks + >>> labels = torch.randint(0, 20, (2, 5), device='cuda') + >>> input_lengths = torch.tensor([10, 8], device='cuda') + >>> label_lengths = torch.tensor([5, 4], device='cuda') + >>> costs = torch.zeros(2, device='cuda') + >>> grads = torch.zeros_like(acts) + >>> big_blank_durations = [2, 4, 8] + >>> success = multiblank_rnnt_loss_gpu(acts, labels, input_lengths, label_lengths, costs, grads, + ... blank_label=0, big_blank_durations=big_blank_durations, + ... fastemit_lambda=0.0, clamp=0.0, num_threads=4, sigma=0.5) + >>> print(success) + True """ minibatch_size = acts.shape[0] diff --git a/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py b/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py index 7054f2abaf5..c27a00f1b4e 100644 --- a/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py +++ b/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py @@ -37,6 +37,21 @@ class _RNNTNumba(Function): + """ + A custom autograd function for computing RNN Transducer (RNNT) loss using Numba. + + This class implements the forward and backward passes for the RNNT loss + computation, leveraging Numba for efficient CPU and GPU implementations. + It is designed to be used within PyTorch's autograd system. + + The class supports both standard RNNT loss and FastEmit regularization, + with options for different reduction methods and gradient clamping. + + Note: + This is an internal class and should not be instantiated directly. + Instead, use the `rnnt_loss` function or `RNNTLossNumba` module. + """ + @staticmethod def forward( ctx, @@ -49,18 +64,34 @@ def forward( fastemit_lambda, clamp, ): - """RNNTNumba Forward. - - log_probs: Tensor of (batch x seqLength x labelLength x outputDim) - containing output from network - labels: 2 dimensional Tensor containing all the targets of - the batch with zero padded - act_lens: Tensor of size (batch) containing size of each - output sequence from the network - label_lens: Tensor of (batch) containing label length of each example - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. + """ + Forward pass for the RNN Transducer loss computation. + + This method computes the RNNT loss given the network outputs and labels. + It supports both CPU and GPU implementations. + + Args: + ctx (object): Context object to save information for backward pass. + acts (torch.Tensor): A 4D tensor (batch x seqLength x labelLength x outputDim) + containing output from network. + labels (torch.Tensor): 2D tensor containing all the targets of the batch + with zero padded. + act_lens (torch.Tensor): 1D tensor of size (batch) containing size of each + output sequence from the network. + label_lens (torch.Tensor): 1D tensor of (batch) containing label length + of each example. + blank (int): The blank label index. + reduction (str): Specifies the reduction to apply to the output. + fastemit_lambda (float): Scaling factor for FastEmit regularization. + clamp (float): Value for gradient clamping. + + Returns: + torch.Tensor: The computed RNNT loss. + + Note: + This method saves gradients in the context for use in the backward pass. + The actual loss computation is delegated to CUDA or CPU implementations + based on the input tensor's device. """ is_cuda = acts.is_cuda @@ -101,15 +132,52 @@ def forward( @staticmethod def backward(ctx, grad_output): + """ + Backward pass for the RNN Transducer loss computation. + + This method computes the gradients of the RNNT loss with respect to the inputs. + + Args: + ctx (object): Context object containing saved tensors from the forward pass. + grad_output (torch.Tensor): Gradient of the loss with respect to the output + of the forward pass. + + Returns: + tuple: A tuple containing the gradients with respect to each input of the + forward function. The gradients for non-tensor inputs are None. + + Note: + This method relies on the gradients computed and saved during the forward pass. + It scales the saved gradients by the incoming gradient and returns them. + The gradient computation is automatically handled by PyTorch's autograd system. + """ if grad_output is not None and ctx.grads is not None: grad_output = grad_output.view(-1, 1, 1, 1).to(ctx.grads) return ctx.grads.mul_(grad_output), None, None, None, None, None, None, None class _MultiblankRNNTNumba(Function): - """Numba class for multi-blank transducer loss + """ + A custom autograd function for computing Multi-blank RNN Transducer (RNNT) loss using Numba. - (https://arxiv.org/pdf/2211.03541.pdf) + This class implements the forward and backward passes for the Multi-blank RNNT loss + computation, leveraging Numba for efficient GPU implementations. It is designed to + be used within PyTorch's autograd system. + + The Multi-blank RNNT loss is an extension of the standard RNNT loss that incorporates + multiple blank symbols with different durations. This approach can improve the + performance of speech recognition systems, especially in streaming scenarios. + + The class supports both standard Multi-blank RNNT loss and FastEmit regularization, + with options for different reduction methods, gradient clamping, and logit + under-normalization. + + Note: + This is an internal class and should not be instantiated directly. + Instead, use the `multiblank_rnnt_loss` function or `MultiblankRNNTLossNumba` module. + + Reference: + https://arxiv.org/pdf/2211.03541.pdf """ @staticmethod @@ -126,15 +194,38 @@ def forward( clamp, sigma, ): - """MultiblankRNNTNumba Forward. - - big_blank_durations: list of durations for multi-blank transducer, e.g. - [2, 4, 8]. - sigma: hyper-parameter for logit under-normalization method for training - multi-blank transducers. Recommended value 0.05. - Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for - the above parameters; - For other parameters for this class, refer to comment for class _RNNTNumba + """ + Forward pass for the Multi-blank RNN Transducer loss computation. + + This method computes the Multi-blank RNNT loss given the network outputs and labels. + It currently supports only GPU implementations. + + Args: + ctx (object): Context object to save information for backward pass. + acts (torch.Tensor): A 4D tensor (batch x seqLength x labelLength x outputDim) + containing output from network. + labels (torch.Tensor): 2D tensor containing all the targets of the batch + with zero padded. + act_lens (torch.Tensor): 1D tensor of size (batch) containing size of each + output sequence from the network. + label_lens (torch.Tensor): 1D tensor of (batch) containing label length + of each example. + blank (int): The standard blank label index. + big_blank_durations (list): List of durations for multi-blank transducer. + reduction (str): Specifies the reduction to apply to the output. + fastemit_lambda (float): Scaling factor for FastEmit regularization. + clamp (float): Value for gradient clamping. + sigma (float): Hyper-parameter for logit under-normalization method. + + Returns: + torch.Tensor: The computed Multi-blank RNNT loss. + + Raises: + NotImplementedError: If attempting to use CPU implementation. + + Note: + This method saves gradients in the context for use in the backward pass. + The actual loss computation is delegated to the GPU implementation. """ is_cuda = acts.is_cuda @@ -181,6 +272,28 @@ def forward( @staticmethod def backward(ctx, grad_output): + """ + Backward pass for the Multi-blank RNN Transducer loss computation. + + This method computes the gradients of the Multi-blank RNNT loss with respect to the inputs. + + Args: + ctx (object): Context object containing saved tensors from the forward pass. + grad_output (torch.Tensor): Gradient of the loss with respect to the output + of the forward pass. + + Returns: + tuple: A tuple containing the gradients with respect to each input of the + forward function. The gradients for non-tensor inputs are None. + + Note: + This method relies on the gradients computed and saved during the forward pass. + It scales the saved gradients by the incoming gradient and returns them. + The gradient computation is automatically handled by PyTorch's autograd system. + + The returned tuple has 11 elements to match the number of inputs in the forward method, + with None values for inputs that don't require gradients. + """ if grad_output is not None and ctx.grads is not None: grad_output = grad_output.view(-1, 1, 1, 1).to(ctx.grads) return ( @@ -208,21 +321,45 @@ def rnnt_loss( fastemit_lambda: float = 0.0, clamp: float = 0.0, ): - """RNN Transducer Loss (functional form) + """ + Compute the RNN Transducer Loss. + + This function calculates the RNN Transducer Loss, which is commonly used in + speech recognition tasks. It supports both CPU and GPU computations. Args: - acts: Tensor of (batch x seqLength x labelLength x outputDim) - containing output from network - labels: 2 dimensional Tensor containing all the targets of - the batch with zero padded - act_lens: Tensor of size (batch) containing size of each - output sequence from the network - label_lens: Tensor of (batch) containing label length of each example - blank (int, optional): blank label. Default: 0. - reduction (string, optional): Specifies the reduction to apply to the output: - 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, - 'mean': the output losses will be divided by the target lengths and - then the mean over the batch is taken. Default: 'mean' + acts (torch.Tensor): A 4D tensor of shape (batch, seqLength, labelLength, outputDim) + containing the output from the network. + labels (torch.Tensor): A 2D tensor containing all the targets of the batch + with zero padding. + act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of + each output sequence from the network. + label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label + length of each example. + blank (int, optional): The blank label index. Defaults to 0. + reduction (str, optional): Specifies the reduction to apply to the output. + Options are 'none', 'mean', or 'sum'. Defaults to 'mean'. + fastemit_lambda (float, optional): Scaling factor for FastEmit regularization. + Defaults to 0.0. + clamp (float, optional): Value for gradient clamping. If positive, gradients + will be clamped to [-clamp, clamp]. Defaults to 0.0. + + Returns: + torch.Tensor: The computed RNN Transducer Loss. + + Raises: + ValueError: If `clamp` is negative. + + Note: + For CPU computations, log_softmax is applied manually, while for GPU + computations, it's computed within the CUDA kernel. + + Example: + >>> acts = torch.randn(2, 10, 5, 20) + >>> labels = torch.randint(0, 19, (2, 5)) + >>> act_lens = torch.tensor([10, 8]) + >>> label_lens = torch.tensor([5, 4]) + >>> loss = rnnt_loss(acts, labels, act_lens, label_lens) """ if not acts.is_cuda: @@ -256,28 +393,52 @@ def multiblank_rnnt_loss( fastemit_lambda: float = 0.0, clamp: float = 0.0, ): - """Multi-blank RNN Transducer (https://arxiv.org/pdf/2211.03541.pdf) + """ + Compute the Multi-blank RNN Transducer Loss. + + This function calculates the Multi-blank RNN Transducer Loss, which is an extension + of the standard RNN Transducer Loss that incorporates multiple blank symbols with + different durations. It is designed to improve the performance of speech recognition + systems, especially for streaming scenarios. - Loss (functional form) Args: - acts: Tensor of (batch x seqLength x labelLength x outputDim) containing - output from network - labels: 2 dimensional Tensor containing all the targets of the batch with - zero padded - act_lens: Tensor of size (batch) containing size of each output - sequence from the network - label_lens: Tensor of (batch) containing label length of each example - blank (int): standard blank label. - big_blank_durations: list of durations for multi-blank transducer, e.g. - [2, 4, 8]. - sigma: hyper-parameter for logit under-normalization method for training - multi-blank transducers. Recommended value 0.05. - Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for - the last two params. - reduction (string, optional): Specifies the reduction to apply to the output: - 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, - 'mean': the output losses will be divided by the target lengths and - then the mean over the batch is taken. Default: 'mean' + acts (torch.Tensor): A 4D tensor of shape (batch, seqLength, labelLength, outputDim) + containing the output from the network. + labels (torch.Tensor): A 2D tensor containing all the targets of the batch + with zero padding. + act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of + each output sequence from the network. + label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label + length of each example. + blank (int): The standard blank label index. + big_blank_durations (list, optional): List of durations for multi-blank transducer, + e.g., [2, 4, 8]. Defaults to an empty list. + reduction (str, optional): Specifies the reduction to apply to the output. + Options are 'none', 'mean', or 'sum'. Defaults to 'mean'. + fastemit_lambda (float, optional): Scaling factor for FastEmit regularization. + Defaults to 0.0. + clamp (float, optional): Value for gradient clamping. If positive, gradients + will be clamped to [-clamp, clamp]. Defaults to 0.0. + + Returns: + torch.Tensor: The computed Multi-blank RNN Transducer Loss. + + Raises: + ValueError: If `clamp` is negative. + NotImplementedError: If trying to use CPU for computation (currently only GPU is supported). + + Note: + This implementation is based on the paper "Multi-blank Transducers for Speech Recognition" + (https://arxiv.org/pdf/2211.03541.pdf). It's designed to work with CUDA-enabled devices. + + Example: + >>> acts = torch.randn(2, 10, 5, 20).cuda() + >>> labels = torch.randint(0, 19, (2, 5)).cuda() + >>> act_lens = torch.tensor([10, 8]).cuda() + >>> label_lens = torch.tensor([5, 4]).cuda() + >>> blank = 0 + >>> big_blank_durations = [2, 4] + >>> loss = multiblank_rnnt_loss(acts, labels, act_lens, label_lens, blank, big_blank_durations) """ if not acts.is_cuda: @@ -309,19 +470,28 @@ def multiblank_rnnt_loss( class RNNTLossNumba(Module): - """RNNT Loss Numba - - Parameters: - blank (int, optional): blank label. Default: 0. - reduction (string, optional): Specifies the reduction to apply to the output: - 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, - 'mean': the output losses will be divided by the target lengths and - then the mean over the batch is taken. Default: 'mean' - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. - clamp: Float value. When set to value >= 0.0, will clamp the - gradient to [-clamp, clamp]. + """ + A PyTorch module for computing RNN Transducer (RNNT) loss using Numba. + + This module provides an efficient implementation of the RNNT loss function, + leveraging Numba for improved performance. It supports both CPU and GPU + computations, with options for different reduction methods, FastEmit + regularization, and gradient clamping. + + The RNNT loss is commonly used in speech recognition tasks, particularly + for training end-to-end models. + + Attributes: + blank (int): The blank label index. Default is 0. + reduction (str): Specifies the reduction to apply to the output. + Can be 'none', 'mean', or 'sum'. Default is 'mean'. + fastemit_lambda (float): Scaling factor for FastEmit regularization. + Default is 0.0. + clamp (float): Value for gradient clamping. When set to a value >= 0.0, + will clamp the gradient to [-clamp, clamp]. Default is -1 (no clamping). + + Note: + This module uses the `_RNNTNumba` function for the actual loss computation. """ def __init__( @@ -335,15 +505,30 @@ def __init__( self.loss = _RNNTNumba.apply def forward(self, acts, labels, act_lens, label_lens): - """Forward RNNTLossNumba. - - log_probs: Tensor of (batch x seqLength x labelLength x outputDim) - containing output from network - labels: 2 dimensional Tensor containing all the targets of the - batch with zero padded - act_lens: Tensor of size (batch) containing size of each output - sequence from the network - label_lens: Tensor of (batch) containing label length of each example + """ + Forward pass for computing the RNN Transducer loss. + + This method calculates the RNNT loss given the network outputs and labels. + It handles both CPU and GPU implementations, applying necessary preprocessing + steps depending on the device. + + Args: + acts (torch.Tensor): A 4D tensor of shape (batch x seqLength x labelLength x outputDim) + containing output from the network. + labels (torch.Tensor): A 2D tensor containing all the targets of the batch + with zero padding. + act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of each + output sequence from the network. + label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label + length of each example. + + Returns: + torch.Tensor: The computed RNNT loss. + + Note: + For CPU computations, this method manually applies log_softmax and handles + gradient clamping if specified. For GPU computations, these operations are + performed within the CUDA kernel for efficiency. """ if not acts.is_cuda: @@ -374,25 +559,32 @@ def forward(self, acts, labels, act_lens, label_lens): class MultiblankRNNTLossNumba(Module): - """Multiblank RNNT Loss Numba - - Parameters: - blank (int): standard blank label. - big_blank_durations: list of durations for multi-blank transducer, e.g. - [2, 4, 8]. - sigma: hyper-parameter for logit under-normalization method for training - multi-blank transducers. Recommended value 0.05. - Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for - the above parameters; - reduction (string, optional): Specifies the reduction to apply to the output: - 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, - 'mean': the output losses will be divided by the target lengths and - then the mean over the batch is taken. Default: 'mean' - fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to - FastEmit: Low-latency Streaming ASR with Sequence-level - Emission Regularization. - clamp: Float value. When set to value >= 0.0, will clamp the - gradient to [-clamp, clamp]. + """ + A PyTorch module for computing Multi-blank RNN Transducer (RNNT) loss using Numba. + + This module implements the Multi-blank RNNT loss function, which is an extension + of the standard RNNT loss that incorporates multiple blank symbols with different + durations. It is designed to improve the performance of speech recognition systems, + especially in streaming scenarios. + + The implementation uses Numba for efficient computation and currently supports + GPU operations only. + + Attributes: + blank (int): The standard blank label index. + big_blank_durations (list): List of durations for multi-blank transducer, e.g., [2, 4, 8]. + reduction (str): Specifies the reduction to apply to the output. + Can be 'none', 'mean', or 'sum'. Default is 'mean'. + fastemit_lambda (float): Scaling factor for FastEmit regularization. Default is 0.0. + clamp (float): Value for gradient clamping. When set to a value >= 0.0, + will clamp the gradient to [-clamp, clamp]. Default is -1 (no clamping). + sigma (float): Hyper-parameter for logit under-normalization method. Default is 0.0. + + Note: + This module uses the `_MultiblankRNNTNumba` function for the actual loss computation. + + Reference: + https://arxiv.org/pdf/2211.03541.pdf """ def __init__( @@ -414,15 +606,33 @@ def __init__( self.sigma = sigma def forward(self, acts, labels, act_lens, label_lens): - """MultiblankRNNTLossNumba Forward. - - log_probs: Tensor of (batch x seqLength x labelLength x outputDim) - containing output from network - labels: 2 dimensional Tensor containing all the targets of - the batch with zero padded - act_lens: Tensor of size (batch) containing size of each output - sequence from the network - label_lens: Tensor of (batch) containing label length of each example + """ + Forward pass for computing the Multi-blank RNN Transducer loss. + + This method calculates the Multi-blank RNNT loss given the network outputs and labels. + It currently supports only GPU implementations, applying necessary preprocessing + steps before the loss computation. + + Args: + acts (torch.Tensor): A 4D tensor of shape (batch x seqLength x labelLength x outputDim) + containing output from the network. + labels (torch.Tensor): A 2D tensor containing all the targets of the batch + with zero padding. + act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of each + output sequence from the network. + label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label + length of each example. + + Returns: + torch.Tensor: The computed Multi-blank RNNT loss. + + Raises: + NotImplementedError: If attempting to use CPU for computation. + + Note: + For GPU computations, this method manually applies log_softmax and handles + gradient clamping if specified. The actual loss computation is performed + within the CUDA kernel for efficiency. """ if not acts.is_cuda: @@ -455,23 +665,115 @@ def forward(self, acts, labels, act_lens, label_lens): def check_type(var, t, name): + """ + Check if a variable has the expected data type. + + This function verifies whether the given variable has the specified data type. + If the variable's type doesn't match the expected type, it raises a TypeError. + + Args: + var (Any): The variable to check. + t (type): The expected data type. + name (str): The name of the variable (used in the error message). + + Raises: + TypeError: If the variable's type doesn't match the expected type. + + Example: + >>> import torch + >>> tensor = torch.tensor([1, 2, 3]) + >>> check_type(tensor, torch.Tensor, "tensor") + >>> check_type(tensor, torch.float32, "tensor") + TypeError: tensor must be torch.float32 + """ if var.dtype is not t: raise TypeError("{} must be {}".format(name, t)) def check_contiguous(var, name): + """ + Check if a tensor is contiguous in memory. + + This function verifies whether the given tensor is contiguous in memory. + If the tensor is not contiguous, it raises a ValueError. + + Args: + var (torch.Tensor): The tensor to check for contiguity. + name (str): The name of the tensor (used in the error message). + + Raises: + ValueError: If the tensor is not contiguous in memory. + + Example: + >>> import torch + >>> tensor = torch.tensor([[1, 2], [3, 4]]) + >>> check_contiguous(tensor, "tensor") + >>> non_contiguous = tensor.t() + >>> check_contiguous(non_contiguous, "non_contiguous") + ValueError: non_contiguous must be contiguous + """ if not var.is_contiguous(): raise ValueError("{} must be contiguous".format(name)) def check_dim(var, dim, name): + """ + Check if a tensor has the expected number of dimensions. + + This function verifies whether the given tensor has the specified number of dimensions. + If the tensor's dimensionality doesn't match the expected value, it raises a ValueError. + + Args: + var (torch.Tensor): The tensor to check. + dim (int): The expected number of dimensions. + name (str): The name of the tensor (used in the error message). + + Raises: + ValueError: If the tensor's number of dimensions doesn't match the expected value. + + Example: + >>> import torch + >>> tensor_2d = torch.tensor([[1, 2], [3, 4]]) + >>> check_dim(tensor_2d, 2, "tensor_2d") + >>> tensor_3d = torch.ones(2, 3, 4) + >>> check_dim(tensor_3d, 2, "tensor_3d") + ValueError: tensor_3d must be 2D + """ if len(var.shape) != dim: raise ValueError("{} must be {}D".format(name, dim)) def certify_inputs(log_probs, labels, lengths, label_lengths): # check_type(log_probs, torch.float32, "log_probs") - check_type(labels, torch.int32, "labels") + """ + Certify that the input tensors meet the required specifications for RNNT loss computation. + + This function performs a series of checks on the input tensors to ensure they meet + the necessary requirements for computing the RNNT loss. It verifies data types, + contiguity, dimensions, and shape consistency. + + Args: + log_probs (torch.Tensor): A 4D tensor of log probabilities from the network output. + labels (torch.Tensor): A 2D tensor of label sequences. + lengths (torch.Tensor): A 1D tensor of input sequence lengths. + label_lengths (torch.Tensor): A 1D tensor of label sequence lengths. + + Raises: + TypeError: If any tensor has an incorrect data type. + ValueError: If any tensor is not contiguous, has incorrect dimensions, + or if there's a mismatch in batch sizes or sequence lengths. + + Note: + This function is typically called internally by the RNNT loss function + to validate inputs before computation. + + Example: + >>> log_probs = torch.randn(2, 10, 5, 20, dtype=torch.float32) + >>> labels = torch.randint(0, 19, (2, 5), dtype=torch.int32) + >>> lengths = torch.tensor([10, 8], dtype=torch.int32) + >>> label_lengths = torch.tensor([5, 4], dtype=torch.int32) + >>> certify_inputs(log_probs, labels, lengths, label_lengths) + """ check_type(label_lengths, torch.int32, "label_lengths") check_type(lengths, torch.int32, "lengths") check_contiguous(log_probs, "log_probs") From c5d9ae7f5882eb3ad4da9fd8b11634cbab28a14d Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 14 Aug 2024 02:52:39 -0400 Subject: [PATCH 05/57] Make easier build for docs --- ci/doc.sh | 21 +++++++++++++++++++++ doc/README.md | 19 +++++++++++++++++-- doc/notebook2rst.sh | 20 +++++++++----------- setup.py | 1 + 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/ci/doc.sh b/ci/doc.sh index fbd6a51afa3..14660f92178 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -1,7 +1,25 @@ #!/usr/bin/env bash +set -euo pipefail + . tools/activate_python.sh +clean_outputs() { + rm -rf dist + rm -rf espnet_bin + rm -rf espnet2_bin + rm -rf utils_py + + rm -rf doc/_gen + rm -rf doc/build + + rm -rf doc/vuepress/src/*.md + rm -rf doc/vuepress/src/notebook + rm -rf doc/vuepress/src/* + rm -rf doc/vuepress/src/.vuepress/.temp + rm -rf doc/vuepress/src/.vuepress/.cache +} + build_and_convert () { # $1: path # $2: output @@ -17,6 +35,9 @@ if [ ! -e tools/kaldi ]; then git clone https://github.com/kaldi-asr/kaldi --depth 1 tools/kaldi fi +# clean previous build +clean_outputs + # build sphinx document under doc/ mkdir -p doc/_gen mkdir -p doc/_gen/tools diff --git a/doc/README.md b/doc/README.md index a316b2998c4..bb94fffb3dd 100644 --- a/doc/README.md +++ b/doc/README.md @@ -5,11 +5,27 @@ We use [sphinx](https://www.sphinx-doc.org) to generate HTML documentation. ```sh +# Clean conda env for docs $ cd +$ conda create -p ./envs python=3.9 +$ conda activate ./envs + +# Requirements $ pip install -e ".[doc]" +$ conda install conda-forge::ffmpeg + +# (Optional requirement) To use flake8-docstrings $ pip install -U flake8-docstrings ``` +If you used the above clean conda environment, you have write your own `. tools/activate_python.sh`. +The example will be: +```sh +#!/usr/bin/env bash + +. /miniconda/etc/profile.d/conda.sh && conda activate /envs +``` + ## Style check using flake8-docstrings You can check that your docstring style is correct by `ci/test_flake8.sh` using [flake8-docstrings](https://pypi.org/project/flake8-docstrings/). @@ -39,8 +55,7 @@ DO NOT ADD NEW FILES TO THIS BLACK LIST! ## Generate HTML -You can generate local HTML manually using sphinx Makefile - +You can generate local HTML manually using sphinx Makefile. ```sh $ cd $ ./ci/doc.sh diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index aa90909a641..89da536c908 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -14,32 +14,30 @@ Jupyter notebooks for course demos and tutorials. " cd notebook -documents=`for i in $(ls -d */); do echo ${i%%/}; done` -for document in "${documents[@]}"; do - echo "## ${document}\n" - find ./${document} \ +for basedir in */; do + echo "## ${basedir}\n" + find ${basedir} \ -type f \ -name '*.ipynb' \ -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --clear-output \"{}\"" \; - find ./${document} \ + find ./${basedir} \ -type f \ -name '*.ipynb' \ -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --to markdown \"{}\"" \; - basedir=./${document} - for md_file in `find "./${document}" -name "*.md"`; do + for md_file in `find ${basedir} -name "*.md"`; do filename=`basename ${md_file}` echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" done - for ipynb_file in `find "./${document}" -name "*.ipynb"`; do + for ipynb_file in `find ${basedir} -name "*.ipynb"`; do rm ${ipynb_file} done # generate README.md - echo "# ${document} Demo" > ./${document}/README.md - for md_file in `find "./${document}" -name "*.md"`; do + echo "# ${basedir} Demo" > ${basedir}README.md + for md_file in `find ${basedir} -name "*.md"`; do filename=`basename ${md_file}` - echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" >> ./${document}/README.md + echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" >> ${basedir}README.md done echo "" done diff --git a/setup.py b/setup.py index 24046aa35ae..128933fdaf3 100644 --- a/setup.py +++ b/setup.py @@ -120,6 +120,7 @@ } requirements["all"].extend(requirements["train"] + requirements["recipe"]) requirements["test"].extend(requirements["train"]) +requirements["doc"].extend(requirements["all"]) install_requires = requirements["install"] setup_requires = requirements["setup"] From ec25a011323459629e7638f1a74e96c864d6bd6c Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 14 Aug 2024 11:51:31 -0400 Subject: [PATCH 06/57] Upgrade Sphinx - removed dependency for recommonmark --- doc/conf.py | 22 ++-------------------- setup.py | 5 +++-- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index c2f5acd1881..e8cbfe0358d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -41,6 +41,7 @@ "sphinx.ext.todo", "sphinxarg.ext", "sphinx_markdown_tables", + 'myst_parser', ] # Add any paths that contain templates here, relative to this directory. @@ -52,31 +53,12 @@ # source_suffix = '.rst' source_suffix = [".rst", ".md"] -# enable to markdown -from recommonmark.parser import CommonMarkParser - source_parsers = { - ".md": CommonMarkParser, + ".md": 'markdown', } -# AutoStructify setting ref: https://qiita.com/pashango2/items/d1b379b699af85b529ce -from recommonmark.transform import AutoStructify - github_doc_root = "https://github.com/rtfd/recommonmark/tree/master/doc/" - -def setup(app): - app.add_config_value( - "recommonmark_config", - { - "url_resolver": lambda url: github_doc_root + url, - "auto_toc_tree_section": "Contents", - }, - True, - ) - app.add_transform(AutoStructify) - - # The master toctree document. master_doc = "index" diff --git a/setup.py b/setup.py index 24046aa35ae..e2dd8721698 100644 --- a/setup.py +++ b/setup.py @@ -107,11 +107,12 @@ ], "doc": [ "Jinja2<3.1", - "Sphinx==2.1.2", + "Sphinx<9.0.0", "sphinx-rtd-theme>=0.2.4", "sphinx-argparse>=0.2.5", "commonmark==0.8.1", - "recommonmark>=0.4.0", + # "recommonmark>=0.4.0", + "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", "jupyter", From 0df02614d41ea677ec94febd1cddc0321a438ff1 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 14 Aug 2024 21:37:49 -0400 Subject: [PATCH 07/57] Bug fixed for divide_lang.sh - Add simlink into the TEMPLATE/asr1 folder - Fixed exit code when called with `--help` --- egs2/TEMPLATE/asr1/utils/divide_lang.sh | 1 + utils/divide_lang.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 egs2/TEMPLATE/asr1/utils/divide_lang.sh diff --git a/egs2/TEMPLATE/asr1/utils/divide_lang.sh b/egs2/TEMPLATE/asr1/utils/divide_lang.sh new file mode 120000 index 00000000000..770a679fe18 --- /dev/null +++ b/egs2/TEMPLATE/asr1/utils/divide_lang.sh @@ -0,0 +1 @@ +../../../../utils/divide_lang.sh \ No newline at end of file diff --git a/utils/divide_lang.sh b/utils/divide_lang.sh index b1fd3c22349..fcb8a56d8bc 100755 --- a/utils/divide_lang.sh +++ b/utils/divide_lang.sh @@ -5,10 +5,10 @@ . ./path.sh -if [ "$#" -ne 2 ]; then +if [ "$1" = "--help" ] ; then echo "Usage: $0 " echo "e.g.: $0 dev" - exit 1 + exit 0 fi set=$1 From faf3d9196901151c059bdeed26f3fc41fd462afb Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 14 Aug 2024 21:50:28 -0400 Subject: [PATCH 08/57] Bug fixed for free-gpu.sh - same fix as divide_lang.sh --- egs2/TEMPLATE/asr1/utils/free-gpu.sh | 1 + utils/free-gpu.sh | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 120000 egs2/TEMPLATE/asr1/utils/free-gpu.sh diff --git a/egs2/TEMPLATE/asr1/utils/free-gpu.sh b/egs2/TEMPLATE/asr1/utils/free-gpu.sh new file mode 120000 index 00000000000..06d1890dc22 --- /dev/null +++ b/egs2/TEMPLATE/asr1/utils/free-gpu.sh @@ -0,0 +1 @@ +../../../../utils/free-gpu.sh \ No newline at end of file diff --git a/utils/free-gpu.sh b/utils/free-gpu.sh index 6db1969dfe9..1d6fba28d5d 100755 --- a/utils/free-gpu.sh +++ b/utils/free-gpu.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Author: Gaurav Kumar - -# Usage: e.g. -# % free-gpu.sh -n 2 -# 1,2 +if [ "$1" = "--help" ] ; then + echo "Usage: $0 -n " + echo "e.g.: $0 -n 2" + echo "1, 2" + exit 0 +fi # Allow requests for multiple GPUs # (Optional) defaults to 1 From b9cd6200b2463bba65d9467aad88bae02f4b4114 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 14 Aug 2024 22:42:13 -0400 Subject: [PATCH 09/57] Bug fixed for sentencepiece document - I noticed that `spm_train` command does not have help. So I removed it. --- ci/doc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/doc.sh b/ci/doc.sh index fbd6a51afa3..e7abbc8eb7c 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -48,7 +48,9 @@ mkdir espnet2_bin mv espnet2_bin ./doc/_gen/tools build_and_convert "utils/*.sh" utils -build_and_convert "tools/sentencepiece_commands/spm_*" spm +build_and_convert "tools/sentencepiece_commands/spm_decode" spm +build_and_convert "tools/sentencepiece_commands/spm_encode" spm +# There seems no help prepared for spm_train command. ./doc/notebook2rst.sh > ./doc/notebooks.md From ba1012a68f49963b2142a937d7ed72a1965b32db Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 14 Aug 2024 23:02:37 -0400 Subject: [PATCH 10/57] Bug fixed for notebook to markdown script --- doc/notebook2rst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index aa90909a641..ac5de17814b 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -15,7 +15,7 @@ Jupyter notebooks for course demos and tutorials. cd notebook documents=`for i in $(ls -d */); do echo ${i%%/}; done` -for document in "${documents[@]}"; do +for document in ${documents[@]}; do echo "## ${document}\n" find ./${document} \ -type f \ From 927a06467941192805b0e2fa187bcebadbb96440 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Thu, 15 Aug 2024 00:06:33 -0400 Subject: [PATCH 11/57] Added __init__.py in espnet/vc and espnet/vc/pytorch_backend --- espnet/vc/__init__.py | 0 espnet/vc/pytorch_backend/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 espnet/vc/__init__.py create mode 100644 espnet/vc/pytorch_backend/__init__.py diff --git a/espnet/vc/__init__.py b/espnet/vc/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/espnet/vc/pytorch_backend/__init__.py b/espnet/vc/pytorch_backend/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 8d39ef06a0b2b876789bac6e9a2fbc60254de711 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Fri, 16 Aug 2024 16:58:35 -0400 Subject: [PATCH 12/57] Added documentation on how this homepage is generated --- doc/about_this_doc.md | 87 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 doc/about_this_doc.md diff --git a/doc/about_this_doc.md b/doc/about_this_doc.md new file mode 100644 index 00000000000..1f3e37d2899 --- /dev/null +++ b/doc/about_this_doc.md @@ -0,0 +1,87 @@ + +# Developer's Guide to the ESPnet Homepage + +This document outlines the process of automatically generating the ESPnet homepage. It provides step-by-step instructions for building the homepage and details the underlying operations during the generation process. + +## Building the Homepage + +1. **Clone the ESPnet Repository**: Begin by cloning the ESPnet repository from GitHub. + +2. **Generate the `activate_python.sh` Script**: + - You can generate this file by running either `setup_anaconda.sh` or `setup_venv.sh`. + - Alternatively, create your own virtual environment and manually write the command to activate it in `activate_python.sh`. + +3. **Run `activate_python.sh`**: Execute this script to activate the Python environment. + +4. **Install the Dependencies**: + Install the necessary dependencies using the following commands: + ``` + espnet[all] + espnet[doc] + k2 + chainer + ``` + +5. **Build the Homepage**: + Run the following script to generate the homepage: + ``` + ./ci/doc.sh + ``` + +## Key Points + +- The homepage is built using VuePress, a static site generator that converts Markdown files into a website. +- The primary function of `ci/doc.sh` is to generate Markdown files for all documentation. + +## Step-by-Step Guide to `ci/doc.sh` + +1. **`build_and_convert` Function**: + This function generates documentation for shell scripts by invoking `./doc/usage2rst.sh` on all scripts in the specified directory (`$1`). The `usage2rst.sh` script executes each script with the `--help` option and saves the output as an RST file in the `$2/.rst` directory. + +2. **Temporary Files Directory**: + All temporary files, including RST files, are stored in the `_gen` directory. + +3. **`./doc/argparse2rst.py` Script**: + This script generates documentation for Python tools located in `espnet/bin`, `espnet2/bin`, and `utils/`. These scripts are executable from the command line, so their documentation is separated from the package information. + +4. **`./doc/notebook2rst.sh` Script**: + This script generates the demo section by pulling the notebook repository and converting Jupyter Notebook (`.ipynb`) files into Markdown. + +5. **`./doc/members2rst.py` Script**: + This script generates RST files for all docstrings. It separates out any docstrings for classes or functions that are not class members and excludes private functions (those starting with `_`). The generated RST files are saved in `./_gen/guide`. + +6. **Sphinx Build Process**: + After copying all necessary files to the `_gen` directory, run `sphinx-build` within `_gen`. Running Sphinx directly in the `doc` directory could cause issues, including potential document corruption. Some files, particularly those ending with `_train` (e.g., `espnet2/bin/asr_train.py`), are excluded from the documentation to avoid errors. + +7. **VuePress Directory Setup**: + Copy the Markdown files from the `doc` directory, along with files generated in steps 4 and 6, into the `vuepress/src` directory. This is where VuePress recognizes the pages for the site. + +8. **Language Support Adjustment**: + VuePress doesn’t support some of the programming languages used in code blocks. To address this, we include a command to replace unsupported language codes with equivalent ones. + +9. **Generate Navigation Files**: + Create the `navbar.yml` and `sidebar.yml` files to define the menus displayed at the top and side of the webpage. For more details, refer to the VuePress-Hope documentation on [navbar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#navbar-config) and [sidebar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#sidebar-config). + +10. **Finalize the Build**: + Install Node.js and the necessary dependencies, then build the homepage. To preview the page, comment out the `docs:build` line and uncomment the `docs:dev` line in the script. + +11. **Create a `clean.sh` Script for Development**: + If you need to clear the build cache during development, you can use the following script: + ```bash + #!/bin/bash + + rm -r dist + rm -r espnet_bin + rm -r espnet2_bin + rm -r utils_py + + rm -r doc/_gen + rm -r doc/build + + rm -r doc/vuepress/src/*.md + find doc/vuepress/src/notebook -type f -exec chmod 777 {} \; + rm -r doc/vuepress/src/notebook + rm -r doc/vuepress/src/* + rm -r doc/vuepress/src/.vuepress/.temp + rm -r doc/vuepress/src/.vuepress/.cache + ``` From 73058f28e023d62f1bcd137fa22b0fda6d5cafa5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 21:03:25 +0000 Subject: [PATCH 13/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/about_this_doc.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/about_this_doc.md b/doc/about_this_doc.md index 1f3e37d2899..b33577e14ad 100644 --- a/doc/about_this_doc.md +++ b/doc/about_this_doc.md @@ -7,13 +7,13 @@ This document outlines the process of automatically generating the ESPnet homepa 1. **Clone the ESPnet Repository**: Begin by cloning the ESPnet repository from GitHub. -2. **Generate the `activate_python.sh` Script**: +2. **Generate the `activate_python.sh` Script**: - You can generate this file by running either `setup_anaconda.sh` or `setup_venv.sh`. - Alternatively, create your own virtual environment and manually write the command to activate it in `activate_python.sh`. 3. **Run `activate_python.sh`**: Execute this script to activate the Python environment. -4. **Install the Dependencies**: +4. **Install the Dependencies**: Install the necessary dependencies using the following commands: ``` espnet[all] @@ -22,7 +22,7 @@ This document outlines the process of automatically generating the ESPnet homepa chainer ``` -5. **Build the Homepage**: +5. **Build the Homepage**: Run the following script to generate the homepage: ``` ./ci/doc.sh @@ -35,37 +35,37 @@ This document outlines the process of automatically generating the ESPnet homepa ## Step-by-Step Guide to `ci/doc.sh` -1. **`build_and_convert` Function**: +1. **`build_and_convert` Function**: This function generates documentation for shell scripts by invoking `./doc/usage2rst.sh` on all scripts in the specified directory (`$1`). The `usage2rst.sh` script executes each script with the `--help` option and saves the output as an RST file in the `$2/.rst` directory. -2. **Temporary Files Directory**: +2. **Temporary Files Directory**: All temporary files, including RST files, are stored in the `_gen` directory. -3. **`./doc/argparse2rst.py` Script**: +3. **`./doc/argparse2rst.py` Script**: This script generates documentation for Python tools located in `espnet/bin`, `espnet2/bin`, and `utils/`. These scripts are executable from the command line, so their documentation is separated from the package information. -4. **`./doc/notebook2rst.sh` Script**: +4. **`./doc/notebook2rst.sh` Script**: This script generates the demo section by pulling the notebook repository and converting Jupyter Notebook (`.ipynb`) files into Markdown. -5. **`./doc/members2rst.py` Script**: +5. **`./doc/members2rst.py` Script**: This script generates RST files for all docstrings. It separates out any docstrings for classes or functions that are not class members and excludes private functions (those starting with `_`). The generated RST files are saved in `./_gen/guide`. -6. **Sphinx Build Process**: +6. **Sphinx Build Process**: After copying all necessary files to the `_gen` directory, run `sphinx-build` within `_gen`. Running Sphinx directly in the `doc` directory could cause issues, including potential document corruption. Some files, particularly those ending with `_train` (e.g., `espnet2/bin/asr_train.py`), are excluded from the documentation to avoid errors. -7. **VuePress Directory Setup**: +7. **VuePress Directory Setup**: Copy the Markdown files from the `doc` directory, along with files generated in steps 4 and 6, into the `vuepress/src` directory. This is where VuePress recognizes the pages for the site. -8. **Language Support Adjustment**: +8. **Language Support Adjustment**: VuePress doesn’t support some of the programming languages used in code blocks. To address this, we include a command to replace unsupported language codes with equivalent ones. -9. **Generate Navigation Files**: +9. **Generate Navigation Files**: Create the `navbar.yml` and `sidebar.yml` files to define the menus displayed at the top and side of the webpage. For more details, refer to the VuePress-Hope documentation on [navbar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#navbar-config) and [sidebar configuration](https://theme-hope.vuejs.press/config/theme/layout.html#sidebar-config). -10. **Finalize the Build**: +10. **Finalize the Build**: Install Node.js and the necessary dependencies, then build the homepage. To preview the page, comment out the `docs:build` line and uncomment the `docs:dev` line in the script. -11. **Create a `clean.sh` Script for Development**: +11. **Create a `clean.sh` Script for Development**: If you need to clear the build cache during development, you can use the following script: ```bash #!/bin/bash From fe0299990c68c7c164e3a3969e23af6756bae38c Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Fri, 16 Aug 2024 21:58:44 -0400 Subject: [PATCH 14/57] Removed unrequired text --- doc/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.md b/doc/index.md index 22ce69d9cad..970fb93edf3 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,4 +1,4 @@ -git s--- +--- home: true icon: /assets/image/espnet.png title: ESPnet From df98fc7cdfd8c97c5d5bb86bf6d62d7c70d7c798 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Fri, 16 Aug 2024 21:59:16 -0400 Subject: [PATCH 15/57] Bug fixed for utils/*.py scripts --- doc/argparse2rst.py | 4 +++- doc/conf.py | 2 +- doc/convert_custom_tags_to_html.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/argparse2rst.py b/doc/argparse2rst.py index d10a00acbd0..5179d49cf58 100755 --- a/doc/argparse2rst.py +++ b/doc/argparse2rst.py @@ -80,6 +80,8 @@ def get_parser(): for m in modinfo: cmd = m.path.name sep = "~" * len(cmd) + mname = m.name if m.name.startswith("espnet") \ + else ".".join(m.name.split(".")[1:]) with open(f"{args.output_dir}/{cmd[:-3]}.rst", "w") as writer: # remove .py writer.write( f""".. _{cmd} @@ -87,7 +89,7 @@ def get_parser(): {sep} .. argparse:: - :module: {m.name} + :module: {mname} :func: get_parser :prog: {cmd} diff --git a/doc/conf.py b/doc/conf.py index e8cbfe0358d..3342d149b4e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -21,7 +21,7 @@ import sys sys.path.insert(0, os.path.abspath("../espnet/nets")) -sys.path.insert(0, os.path.abspath("../utils")) +sys.path.insert(0, os.path.abspath("../../utils")) # -- General configuration ------------------------------------------------ diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py index 63f39de7762..0e6f7572e51 100644 --- a/doc/convert_custom_tags_to_html.py +++ b/doc/convert_custom_tags_to_html.py @@ -147,7 +147,7 @@ def get_parser(): def replace_custom_tags(content): # Regex to find tags and their content - tag_pattern = re.compile(r'<(?!\/)(?!\!!-!-)(.+?)(?!\/)>') + tag_pattern = re.compile(r'<(?!!--)([^>]+)>') def replace_tag(match): tag_name = match.group(1) if len(tag_name) > 50: From 61d9348b974f25283ffa31ca01d2b9f277d1915f Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Fri, 16 Aug 2024 22:19:01 -0400 Subject: [PATCH 16/57] Removed unrequired comment out --- .../asr/transducer/beam_search_transducer.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/espnet2/asr/transducer/beam_search_transducer.py b/espnet2/asr/transducer/beam_search_transducer.py index b0e269dc551..4d5aa99cfed 100644 --- a/espnet2/asr/transducer/beam_search_transducer.py +++ b/espnet2/asr/transducer/beam_search_transducer.py @@ -19,17 +19,17 @@ @dataclass -# class Hypothesis: -# """Default hypothesis definition for Transducer search algorithms.""" - -# score: float -# yseq: List[int] -# dec_state: Union[ -# Tuple[torch.Tensor, Optional[torch.Tensor]], -# List[Optional[torch.Tensor]], -# torch.Tensor, -# ] -# lm_state: Union[Dict[str, Any], List[Any]] = None +class Hypothesis: + """Default hypothesis definition for Transducer search algorithms.""" + + score: float + yseq: List[int] + dec_state: Union[ + Tuple[torch.Tensor, Optional[torch.Tensor]], + List[Optional[torch.Tensor]], + torch.Tensor, + ] + lm_state: Union[Dict[str, Any], List[Any]] = None @dataclass From 57e60a24deedce478382c9544f1a8c82fe139103 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Sat, 17 Aug 2024 00:14:16 -0400 Subject: [PATCH 17/57] Removed unrequired line --- ci/doc.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/doc.sh b/ci/doc.sh index e7abbc8eb7c..ee41f56f960 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -59,9 +59,6 @@ python ./doc/members2rst.py --root espnet --dst ./doc/_gen/guide --exclude espne python ./doc/members2rst.py --root espnet2 --dst ./doc/_gen/guide --exclude espnet2.bin python ./doc/members2rst.py --root espnetez --dst ./doc/_gen/guide -# generate package doc -./doc/module2rst.py --root espnet espnet2 --dst ./doc --exclude espnet.bin - # build markdown cp ./doc/index.rst ./doc/_gen/index.rst cp ./doc/conf.py ./doc/_gen/ From 18dd1d669ef3f5972c5a2c9e65bba915768c791c Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Sat, 17 Aug 2024 00:15:04 -0400 Subject: [PATCH 18/57] Bugfix for generating html. - Bugfix for class with `@dataclass` annotation --- doc/convert_md_to_homepage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/convert_md_to_homepage.py b/doc/convert_md_to_homepage.py index c9372877499..79ed1172c5f 100644 --- a/doc/convert_md_to_homepage.py +++ b/doc/convert_md_to_homepage.py @@ -23,7 +23,7 @@ def small_bracket(text): # forward_core(nnet_output, ys, hlens, ylens) # -> forward_core(nnet_output, ys, hlens, ylens)" text = text.replace("<", "<").replace(">", ">") - brackets = re.findall(r'\(([^)]+)\)', text) + brackets = re.findall(r'\((.*)\)', text) if len(brackets) > 0: text = text.replace( f"({brackets[0]})", From 2cd8ae0966ac9b7ff874d0027cd8d02e812d82fa Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Sat, 17 Aug 2024 04:49:26 -0400 Subject: [PATCH 19/57] Fix bug, add doc --- doc/README.md | 1 + doc/notebook2rst.sh | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/README.md b/doc/README.md index bb94fffb3dd..7adaa451353 100644 --- a/doc/README.md +++ b/doc/README.md @@ -13,6 +13,7 @@ $ conda activate ./envs # Requirements $ pip install -e ".[doc]" $ conda install conda-forge::ffmpeg +$ conda install conda-forge::nodejs # (Optional requirement) To use flake8-docstrings $ pip install -U flake8-docstrings diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index 89da536c908..9fe4edce87b 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -8,14 +8,14 @@ if [ ! -d notebook ]; then git clone https://github.com/espnet/notebook --depth 1 fi -echo "# Notebook +echo "# Notebook Jupyter notebooks for course demos and tutorials. " cd notebook for basedir in */; do - echo "## ${basedir}\n" + printf "## ${basedir}\n" find ${basedir} \ -type f \ -name '*.ipynb' \ @@ -27,7 +27,7 @@ for basedir in */; do for md_file in `find ${basedir} -name "*.md"`; do filename=`basename ${md_file}` - echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" + echo "* [${filename}](./${md_file:((${#basedir})):100})" done for ipynb_file in `find ${basedir} -name "*.ipynb"`; do rm ${ipynb_file} @@ -37,7 +37,7 @@ for basedir in */; do echo "# ${basedir} Demo" > ${basedir}README.md for md_file in `find ${basedir} -name "*.md"`; do filename=`basename ${md_file}` - echo "* [${filename}](./${md_file:((${#basedir}+1)):100})" >> ${basedir}README.md + echo "* [${filename}](./${md_file:((${#basedir})):100})" >> ${basedir}README.md done echo "" done From d1e7d28eafd5ac396db497fc2167f85efdc6cd37 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Sat, 17 Aug 2024 04:55:50 -0400 Subject: [PATCH 20/57] clarify dependencies --- .github/workflows/publish_doc.yml | 2 +- ci/doc.sh | 12 ------------ setup.py | 8 ++------ 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index b38e6b1703c..a99ec974df4 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -47,7 +47,7 @@ jobs: - name: install dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc + sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc nodejs npm - name: install espnet env: ESPNET_PYTHON_VERSION: 3.8 diff --git a/ci/doc.sh b/ci/doc.sh index b238b402560..6b84d052367 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -111,18 +111,6 @@ python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/tools/ cd ./doc/vuepress python create_menu.py --root ./src -# check if node is installed -if which node > /dev/null -then - echo "node is installed, skipping..." -else - apt install -y nodejs npm - npm install n -g - n stable - apt purge -y nodejs npm - apt autoremove -y -fi - npm i # npm run docs:dev npm run docs:build diff --git a/setup.py b/setup.py index 7d5d1674677..d50833a7831 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,6 @@ "asteroid_filterbanks==0.4.0", # UASR "editdistance", - # fix CI error due to the use of deprecated functions - # https://github.com/espnet/espnet/actions/runs/3174416926/jobs/5171182884#step:8:8419 - # https://importlib-metadata.readthedocs.io/en/latest/history.html#v5-0-0 - "importlib-metadata<5.0", ], # train: The modules invoked when training only. "train": [ @@ -107,16 +103,16 @@ ], "doc": [ "Jinja2<3.1", - "Sphinx<9.0.0", + "sphinx<9.0.0", "sphinx-rtd-theme>=0.2.4", "sphinx-argparse>=0.2.5", "commonmark==0.8.1", - # "recommonmark>=0.4.0", "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", "jupyter", "sphinx-markdown-builder", + "importlib-metadata", ], } requirements["all"].extend(requirements["train"] + requirements["recipe"]) From 3ea810f81cba025ac42633f2c335c3d86a0daaeb Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Sat, 17 Aug 2024 05:01:57 -0400 Subject: [PATCH 21/57] Add readme --- doc/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/README.md b/doc/README.md index 7adaa451353..2c40bf094ca 100644 --- a/doc/README.md +++ b/doc/README.md @@ -7,7 +7,7 @@ We use [sphinx](https://www.sphinx-doc.org) to generate HTML documentation. ```sh # Clean conda env for docs $ cd -$ conda create -p ./envs python=3.9 +$ conda create -p ./envs python=3.8 $ conda activate ./envs # Requirements @@ -56,14 +56,13 @@ DO NOT ADD NEW FILES TO THIS BLACK LIST! ## Generate HTML -You can generate local HTML manually using sphinx Makefile. +You can generate and test the webpage using sphinx Makefile. ```sh $ cd $ ./ci/doc.sh +$ npm run docs:dev ``` -open `doc/build/index.html` - ## Deploy When your PR is merged into `master` branch, our [CI](https://github.com/espnet/espnet/blob/master/.github/workflows/doc.yml) will automatically deploy your sphinx html into https://espnet.github.io/espnet/. From 866a5ed542fbc8959cc865165872ca377854cbf0 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Sun, 18 Aug 2024 09:25:03 -0400 Subject: [PATCH 22/57] Updated python version from 3.8 to 3.10 for document generation --- .github/workflows/publish_doc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index b38e6b1703c..3b24f66d132 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -40,7 +40,7 @@ jobs: key: pip-${{ hashFiles('**/setup.py') }} - uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.10 architecture: "x64" - name: check OS run: cat /etc/os-release @@ -50,7 +50,7 @@ jobs: sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc - name: install espnet env: - ESPNET_PYTHON_VERSION: 3.8 + ESPNET_PYTHON_VERSION: 3.10 TH_VERSION: 2.0.1 CHAINER_VERSION: 6.0.0 USE_CONDA: false From f477b7e9f0317376c4808e21af9cb541f1e979ee Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Sun, 18 Aug 2024 10:10:57 -0400 Subject: [PATCH 23/57] Updated workflow for generate document - Update setup_python from v1 to v5 - Removed ESPNET_PYTHON_VERSION as it is not used when USE_CONDA=false --- .github/workflows/publish_doc.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index 19f36edad53..655b4fdd914 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -38,9 +38,9 @@ jobs: with: path: ~/.cache/pip key: pip-${{ hashFiles('**/setup.py') }} - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' architecture: "x64" - name: check OS run: cat /etc/os-release @@ -50,7 +50,6 @@ jobs: sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc nodejs npm - name: install espnet env: - ESPNET_PYTHON_VERSION: 3.10 TH_VERSION: 2.0.1 CHAINER_VERSION: 6.0.0 USE_CONDA: false From 959f9a88bf1e5cd264c2d8fc001caaeda3714564 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Mon, 19 Aug 2024 23:38:24 -0400 Subject: [PATCH 24/57] Refactoring doc.sh --- ci/doc.sh | 6 +----- doc/convert_custom_tags_to_html.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ci/doc.sh b/ci/doc.sh index 6b84d052367..2df76a1bb43 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -95,11 +95,7 @@ cp -r ./doc/image ./doc/vuepress/src/ # Document generation has finished. # From the following point we modify files for VuePress. -# Replace language tags to supported language tags -find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```default/```text/g' {} \; -find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```pycon/```python/g' {} \; -find ./doc/vuepress/src/ -name "*.md" -exec sed -i 's/```cd/```text/g' {} \; - +# replace language tags which is not supported by VuePress # And convert custom tags to < and >, as can be recognized a html tag. python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/ diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py index 0e6f7572e51..8ffa93517d4 100644 --- a/doc/convert_custom_tags_to_html.py +++ b/doc/convert_custom_tags_to_html.py @@ -131,6 +131,12 @@ "wbr", ] +LANGUAGE_TAG_SET = [ + ("default", "text"), + ("pycon", "python"), + ("cd", "text"), +] + def get_parser(): parser = configargparse.ArgumentParser( description="Convert custom tags to markdown", @@ -198,6 +204,13 @@ def replace_tag(match): return tag_pattern.sub(replace_tag, content) +def replace_language_tags(content): + for (label, lang) in LANGUAGE_TAG_SET: + content = content.replace(f"```{label}", f"```{lang}") + + return content + + # parser args = get_parser().parse_args() @@ -209,6 +222,7 @@ def replace_tag(match): # if the tag is not in ALL_HTML_TAGS and does not have its end tag # we need to apply this two functions because # there are custom tags like: "" + content = replace_language_tags(content) content = replace_string_tags(content) content = replace_custom_tags(content) @@ -222,6 +236,7 @@ def replace_tag(match): # Replace the "" and "" with "<" and ">", respectively # if the tag is not in ALL_HTML_TAGS + content = replace_language_tags(content) content = replace_string_tags(content) content = replace_custom_tags(content) From fac024b39f905ed53995f4a0f03d7e08337dd569 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Mon, 19 Aug 2024 23:39:39 -0400 Subject: [PATCH 25/57] Add link to the source code - It refers to the commit ID that the home page is created. --- doc/argparse2rst.py | 10 ++++++++++ doc/members2rst.py | 24 ++++++++++++++++++++---- doc/usage2rst.sh | 5 +++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/argparse2rst.py b/doc/argparse2rst.py index 5179d49cf58..3fef85fb1c7 100755 --- a/doc/argparse2rst.py +++ b/doc/argparse2rst.py @@ -4,10 +4,16 @@ import pathlib import re import os +import subprocess import configargparse + +def get_git_revision_hash() -> str: + return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() + + class ModuleInfo: def __init__(self, path): self.path = pathlib.Path(path) @@ -79,6 +85,8 @@ def get_parser(): # print argparse to each files for m in modinfo: cmd = m.path.name + sourceurl = f"https://github.com/espnet/espnet/blob/" \ + + get_git_revision_hash() + str(m.path.parent / m.path.stem) + ".py" sep = "~" * len(cmd) mname = m.name if m.name.startswith("espnet") \ else ".".join(m.name.split(".")[1:]) @@ -88,6 +96,8 @@ def get_parser(): {cmd} {sep} +`source <{sourceurl}>`_ + .. argparse:: :module: {mname} :func: get_parser diff --git a/doc/members2rst.py b/doc/members2rst.py index 1e72a77788a..68a2586fc78 100644 --- a/doc/members2rst.py +++ b/doc/members2rst.py @@ -4,10 +4,18 @@ import os import ast import sys +import subprocess import configargparse +def get_git_revision_hash() -> str: + return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() + + +GIT_HASH = get_git_revision_hash() + + def to_module(path_name): ret = path_name.replace(".py", "").replace("/", ".") if ret.endswith("."): @@ -31,20 +39,28 @@ def parse_ast(filename): return ast.parse(file.read(), filename=filename) -def gen_func_rst(func_name, writer): +def gen_func_rst(func_name, writer, filepath, lineno): + sourceurl = f"https://github.com/espnet/espnet/blob/" \ + + GIT_HASH + filepath + f":L{lineno}" writer.write(f""".. _{func_name} {func_name} {"~" * len(func_name)} +`source <{sourceurl}>`_ + .. autofunction:: {func_name} """) -def gen_class_rst(class_name, writer): +def gen_class_rst(class_name, writer, filepath, lineno): + sourceurl = f"https://github.com/espnet/espnet/blob/" \ + + GIT_HASH + filepath + f":L{lineno}" writer.write(f""".. _{class_name} {class_name} {"~" * len(class_name)} +`source <{sourceurl}>`_ + .. autoclass:: {class_name} :members: :undoc-members: @@ -89,7 +105,7 @@ def gen_class_rst(class_name, writer): print(f"[INFO] generating {func.name} in {module_name}") # 1.2 generate RST with open(f"{gendir}/{args.root}/{submodule_name}/{function_name}.rst", "w") as f_rst: - gen_func_rst(f"{module_name}.{function_name}", f_rst) + gen_func_rst(f"{module_name}.{function_name}", f_rst, p, func.lineno) # 2 get classes for clz in top_level_classes(parse_ast(p).body): @@ -97,4 +113,4 @@ def gen_class_rst(class_name, writer): print(f"[INFO] generating {clz.name} in {module_name}") # 1.2 generate RST with open(f"{gendir}/{args.root}/{submodule_name}/{class_name}.rst", "w") as f_rst: - gen_class_rst(f"{module_name}.{class_name}", f_rst) + gen_class_rst(f"{module_name}.{class_name}", f_rst, p, clz.lineno) diff --git a/doc/usage2rst.sh b/doc/usage2rst.sh index 56c47ed0160..96b88b81f0a 100755 --- a/doc/usage2rst.sh +++ b/doc/usage2rst.sh @@ -8,6 +8,8 @@ if [ $1 == "--help" ]; then fi real=$(realpath $1) +# githash=`git rev-parse HEAD` +githash=8574aa8e0d5968916df1df42c28621a48b2a5281 cd ./egs2/wsj/asr1 . path.sh @@ -17,12 +19,15 @@ len=${#cmd} r=$(dirname $real) sep=$(printf '~%.0s' $(seq $len)) usage=$($real --help |& sed "s?${r}/??g" | grep -v -e '--help' | sed "s/^/ /g") +sourceurl="https://github.com/espnet/espnet/blob/${githash}/$1" cat <\`_ + .. code-block:: none ${usage} From 93fe0795fd2e665adf569628cb1971a22cfb059a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 03:41:13 +0000 Subject: [PATCH 26/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/convert_custom_tags_to_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py index 8ffa93517d4..9dee9e38819 100644 --- a/doc/convert_custom_tags_to_html.py +++ b/doc/convert_custom_tags_to_html.py @@ -207,7 +207,7 @@ def replace_tag(match): def replace_language_tags(content): for (label, lang) in LANGUAGE_TAG_SET: content = content.replace(f"```{label}", f"```{lang}") - + return content From 85d5de4a1e5c88e0d74e5dd10944cd886ae2d197 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 20 Aug 2024 23:03:18 -0400 Subject: [PATCH 27/57] Limit the tag-conversion script to sphinx-generated markdown. - sometimes `<>` should be converted to `<` or `>`. This process might not be required for tools or other markdown files. - This process can fix text corruption on landing page and ci issue in spm_decode. --- ci/doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/doc.sh b/ci/doc.sh index 2df76a1bb43..fd244ee3427 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -97,7 +97,7 @@ cp -r ./doc/image ./doc/vuepress/src/ # From the following point we modify files for VuePress. # replace language tags which is not supported by VuePress # And convert custom tags to < and >, as can be recognized a html tag. -python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/ +python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/guide # Convert API document to specific html tags to display sphinx style python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/guide/ From 1b59fa9426c468a0462f730adcbf5e48501ea0ae Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 21 Aug 2024 00:22:54 -0400 Subject: [PATCH 28/57] ci bug fixed --- ci/doc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/doc.sh b/ci/doc.sh index fd244ee3427..fd8e373f110 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -98,6 +98,7 @@ cp -r ./doc/image ./doc/vuepress/src/ # replace language tags which is not supported by VuePress # And convert custom tags to < and >, as can be recognized a html tag. python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/guide +python ./doc/convert_custom_tags_to_html.py ./doc/vuepress/src/tools # Convert API document to specific html tags to display sphinx style python ./doc/convert_md_to_homepage.py ./doc/vuepress/src/guide/ From be79d3ec08bb237e6cff8427f3b0d57234d6d7b9 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Thu, 22 Aug 2024 00:45:46 -0400 Subject: [PATCH 29/57] CI bug fixed - It seems that the usage2rst had the bug for "missing end tag". --- .github/workflows/publish_doc.yml | 3 ++- ci/doc.sh | 2 +- doc/usage2rst.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 .github/workflows/publish_doc.yml diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml old mode 100644 new mode 100755 index 655b4fdd914..e6c9117e45d --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -50,6 +50,7 @@ jobs: sudo apt-get install -qq -y cmake python3-dev git pandoc ffmpeg bc nodejs npm - name: install espnet env: + ESPNET_PYTHON_VERSION: 3.10 TH_VERSION: 2.0.1 CHAINER_VERSION: 6.0.0 USE_CONDA: false @@ -61,4 +62,4 @@ jobs: uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: doc/build + publish_dir: ./dist diff --git a/ci/doc.sh b/ci/doc.sh index fd8e373f110..5bf7f84e31a 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -113,4 +113,4 @@ npm i npm run docs:build mv src/.vuepress/dist ../../ -touch doc/build/.nojekyll +touch dist/.nojekyll diff --git a/doc/usage2rst.sh b/doc/usage2rst.sh index 96b88b81f0a..2cfa7a30098 100755 --- a/doc/usage2rst.sh +++ b/doc/usage2rst.sh @@ -21,7 +21,7 @@ sep=$(printf '~%.0s' $(seq $len)) usage=$($real --help |& sed "s?${r}/??g" | grep -v -e '--help' | sed "s/^/ /g") sourceurl="https://github.com/espnet/espnet/blob/${githash}/$1" cat < Date: Thu, 22 Aug 2024 07:13:07 -0400 Subject: [PATCH 30/57] fix minor bug --- ci/doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/doc.sh b/ci/doc.sh index 5bf7f84e31a..30e795edccb 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -113,4 +113,4 @@ npm i npm run docs:build mv src/.vuepress/dist ../../ -touch dist/.nojekyll +touch ../../dist/.nojekyll From e0e8f0e6a1b126dda3d0151b4e36828adce0c57a Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Thu, 22 Aug 2024 07:13:19 -0400 Subject: [PATCH 31/57] Update readme --- doc/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/README.md b/doc/README.md index 2c40bf094ca..38278f9530e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -7,13 +7,13 @@ We use [sphinx](https://www.sphinx-doc.org) to generate HTML documentation. ```sh # Clean conda env for docs $ cd -$ conda create -p ./envs python=3.8 +$ conda create -p ./envs python=3.10 $ conda activate ./envs # Requirements $ pip install -e ".[doc]" $ conda install conda-forge::ffmpeg -$ conda install conda-forge::nodejs +$ conda install conda-forge::nodejs==22.6.0 # (Optional requirement) To use flake8-docstrings $ pip install -U flake8-docstrings @@ -24,6 +24,7 @@ The example will be: ```sh #!/usr/bin/env bash +# You might check $CONDA_EXE to find the . /miniconda/etc/profile.d/conda.sh && conda activate /envs ``` From 1ac669a98c30e28620ca5e3c13167a9bac039d43 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Thu, 22 Aug 2024 07:13:35 -0400 Subject: [PATCH 32/57] Make ci happy --- doc/notebook2rst.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index 9fe4edce87b..eeb5aed3aa0 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -15,29 +15,23 @@ Jupyter notebooks for course demos and tutorials. cd notebook for basedir in */; do - printf "## ${basedir}\n" + printf "## %s\n" "${basedir}" find ${basedir} \ -type f \ -name '*.ipynb' \ - -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --clear-output \"{}\"" \; + -exec bash -c '. ../../tools/activate_python.sh;jupyter nbconvert --clear-output "$1"' shell {} \; find ./${basedir} \ -type f \ -name '*.ipynb' \ - -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --to markdown \"{}\"" \; - - for md_file in `find ${basedir} -name "*.md"`; do - filename=`basename ${md_file}` - echo "* [${filename}](./${md_file:((${#basedir})):100})" - done - for ipynb_file in `find ${basedir} -name "*.ipynb"`; do - rm ${ipynb_file} - done + -exec bash -c '. ../../tools/activate_python.sh;jupyter nbconvert --to markdown "$1"' shell {} \; # generate README.md echo "# ${basedir} Demo" > ${basedir}README.md - for md_file in `find ${basedir} -name "*.md"`; do + find "$basedir" -type f -name "*.md" | while read -r md_file; do filename=`basename ${md_file}` - echo "* [${filename}](./${md_file:((${#basedir})):100})" >> ${basedir}README.md + line="- [${filename}](.${md_file:((${#basedir})):100})" + echo $line + echo $line >> ${basedir}README.md done echo "" done From 676b225d44403fb81dead7a56f80211e1ae455e4 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Mon, 26 Aug 2024 14:14:47 -0400 Subject: [PATCH 33/57] Fixed CI bug --- doc/notebook2rst.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/doc/notebook2rst.sh b/doc/notebook2rst.sh index 9fe4edce87b..11967f6cbf0 100755 --- a/doc/notebook2rst.sh +++ b/doc/notebook2rst.sh @@ -8,6 +8,8 @@ if [ ! -d notebook ]; then git clone https://github.com/espnet/notebook --depth 1 fi +. ../tools/activate_python.sh + echo "# Notebook Jupyter notebooks for course demos and tutorials. @@ -15,29 +17,30 @@ Jupyter notebooks for course demos and tutorials. cd notebook for basedir in */; do - printf "## ${basedir}\n" + printf '## %s\n' "$basedir" find ${basedir} \ -type f \ -name '*.ipynb' \ - -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --clear-output \"{}\"" \; + -exec bash -c 'jupyter nbconvert --clear-output "$1"' shell {} \; find ./${basedir} \ -type f \ -name '*.ipynb' \ - -exec bash -c ". ../../tools/activate_python.sh;jupyter nbconvert --to markdown \"{}\"" \; + -exec bash -c 'jupyter nbconvert --to markdown "$1"' shell {} \; - for md_file in `find ${basedir} -name "*.md"`; do - filename=`basename ${md_file}` + while IFS= read -r -d '' md_file; do + filename=$(basename ${md_file}) echo "* [${filename}](./${md_file:((${#basedir})):100})" - done - for ipynb_file in `find ${basedir} -name "*.ipynb"`; do + done < <(find ${basedir} -name "*.md" -print0) + + while IFS= read -r -d '' ipynb_file; do rm ${ipynb_file} - done + done < <(find ${basedir} -name "*.ipynb" -print0) # generate README.md echo "# ${basedir} Demo" > ${basedir}README.md - for md_file in `find ${basedir} -name "*.md"`; do - filename=`basename ${md_file}` + while IFS= read -r -d '' md_file; do + filename=$(basename ${md_file}) echo "* [${filename}](./${md_file:((${#basedir})):100})" >> ${basedir}README.md - done + done < <(find ${basedir} -name "*.md" -print0) echo "" done From f8346624908b440f4efb4933eff508517e4110ad Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Mon, 26 Aug 2024 14:32:09 -0400 Subject: [PATCH 34/57] Revert "Add or reformat docstrings by LLM" This reverts commit 2c761d5adcdf67aa31c7b85c93bb1ba48728596f. --- espnet2/asr/bayes_risk_ctc.py | 218 +-- espnet2/asr/ctc.py | 178 +-- espnet2/asr/decoder/abs_decoder.py | 59 - .../hugging_face_transformers_decoder.py | 201 +-- espnet2/asr/decoder/mlm_decoder.py | 90 +- espnet2/asr/decoder/rnn_decoder.py | 262 ---- espnet2/asr/decoder/s4_decoder.py | 168 +-- espnet2/asr/decoder/transducer_decoder.py | 292 +--- espnet2/asr/decoder/transformer_decoder.py | 508 ++----- espnet2/asr/decoder/whisper_decoder.py | 234 +-- espnet2/asr/discrete_asr_espnet_model.py | 71 +- espnet2/asr/encoder/abs_encoder.py | 93 -- espnet2/asr/encoder/branchformer_encoder.py | 120 +- espnet2/asr/encoder/conformer_encoder.py | 97 +- .../contextual_block_conformer_encoder.py | 1271 ++++++++--------- .../contextual_block_transformer_encoder.py | 108 +- espnet2/asr/encoder/e_branchformer_encoder.py | 144 +- espnet2/asr/encoder/hubert_encoder.py | 405 ++---- .../hugging_face_transformers_encoder.py | 97 +- espnet2/asr/encoder/linear_encoder.py | 90 +- espnet2/asr/encoder/longformer_encoder.py | 113 +- espnet2/asr/encoder/rnn_encoder.py | 85 +- espnet2/asr/encoder/transformer_encoder.py | 102 +- .../encoder/transformer_encoder_multispkr.py | 110 +- espnet2/asr/encoder/vgg_rnn_encoder.py | 76 +- espnet2/asr/encoder/wav2vec2_encoder.py | 135 +- espnet2/asr/encoder/whisper_encoder.py | 181 +-- espnet2/asr/espnet_model.py | 157 +- espnet2/asr/frontend/abs_frontend.py | 74 - espnet2/asr/frontend/asteroid_frontend.py | 93 +- espnet2/asr/frontend/default.py | 99 +- espnet2/asr/frontend/fused.py | 79 - espnet2/asr/frontend/melspec_torch.py | 75 +- espnet2/asr/frontend/s3prl.py | 94 +- espnet2/asr/frontend/whisper.py | 134 +- espnet2/asr/frontend/windowing.py | 88 +- espnet2/asr/layers/cgmlp.py | 164 +-- espnet2/asr/layers/fastformer.py | 125 +- espnet2/asr/maskctc_model.py | 215 +-- espnet2/asr/pit_espnet_model.py | 234 +-- espnet2/asr/postencoder/abs_postencoder.py | 66 - .../hugging_face_transformers_postencoder.py | 92 +- .../postencoder/length_adaptor_postencoder.py | 74 +- espnet2/asr/preencoder/abs_preencoder.py | 83 -- espnet2/asr/preencoder/linear.py | 76 +- espnet2/asr/preencoder/sinc.py | 169 +-- espnet2/asr/specaug/abs_specaug.py | 54 +- espnet2/asr/specaug/specaug.py | 64 +- espnet2/asr/state_spaces/attention.py | 186 +-- espnet2/asr/state_spaces/base.py | 537 +------ espnet2/asr/state_spaces/block.py | 156 +- espnet2/asr/state_spaces/cauchy.py | 231 +-- espnet2/asr/state_spaces/components.py | 589 +------- espnet2/asr/state_spaces/ff.py | 88 -- espnet2/asr/state_spaces/model.py | 194 +-- espnet2/asr/state_spaces/pool.py | 653 +-------- espnet2/asr/state_spaces/residual.py | 293 +--- espnet2/asr/state_spaces/s4.py | 871 +---------- espnet2/asr/state_spaces/utils.py | 247 +--- espnet2/asr/transducer/error_calculator.py | 170 +-- .../asr/transducer/rnnt_multi_blank/rnnt.py | 149 +- .../rnnt_multi_blank/rnnt_multi_blank.py | 516 ++----- 62 files changed, 1870 insertions(+), 10827 deletions(-) diff --git a/espnet2/asr/bayes_risk_ctc.py b/espnet2/asr/bayes_risk_ctc.py index 7554a7941b7..bc415d95df0 100644 --- a/espnet2/asr/bayes_risk_ctc.py +++ b/espnet2/asr/bayes_risk_ctc.py @@ -9,37 +9,6 @@ class BayesRiskCTC(torch.nn.Module): - """ - Implements the Bayes Risk Connectionist Temporal Classification (BRCTC) loss. - - This class extends torch.nn.Module to provide a BRCTC loss implementation - based on the paper "Bayes Risk CTC: Efficient Sequence Labelling with Neural Networks" - (https://openreview.net/forum?id=Bd7GueaTxUz). - - BRCTC introduces a risk-based approach to CTC, allowing for more flexible - and potentially improved sequence labeling in tasks such as speech recognition. - - Attributes: - risk_strategy (str): Strategy for calculating risk. Can be 'exp' or 'exp_rel'. - group_strategy (str): Strategy for grouping. Can be 'end' or 'end_mean'. - risk_factor (float): Factor controlling the influence of the risk in the loss calculation. - - Args: - risk_strategy (str, optional): Risk calculation strategy. Defaults to 'exp'. - group_strategy (str, optional): Grouping strategy. Defaults to 'end'. - risk_factor (float, optional): Risk factor. Defaults to 0.0. - - Raises: - AssertionError: If an invalid risk_strategy or group_strategy is provided. - - Note: - This implementation requires the K2 library for finite-state automata operations. - - Example: - >>> brctc = BayesRiskCTC(risk_strategy='exp', group_strategy='end', risk_factor=0.1) - >>> loss = brctc(nnet_output, ys_pad, hlens, ylens) - """ - def __init__(self, risk_strategy="exp", group_strategy="end", risk_factor=0.0): super().__init__() @@ -51,39 +20,6 @@ def __init__(self, risk_strategy="exp", group_strategy="end", risk_factor=0.0): self.risk_factor = risk_factor def forward(self, nnet_output, ys_pad, hlens, ylens): - """ - Compute the Bayes Risk CTC loss for a batch of sequences. - - This method implements the forward pass of the Bayes Risk CTC loss calculation. - It handles reordering and filtering of input sequences, and delegates the core - loss computation to the forward_core method. - - Args: - nnet_output (torch.Tensor): Output from the neural network, shape (B, T, C), - where B is batch size, T is the maximum sequence length, and C is the number of classes. - ys_pad (torch.Tensor): Padded target sequences, shape (B, U), - where U is the maximum target sequence length. - hlens (torch.Tensor): Lengths of input sequences, shape (B,). - ylens (torch.Tensor): Lengths of target sequences, shape (B,). - - Returns: - torch.Tensor: The computed loss for each sequence in the batch, shape (B,). - - Note: - This method reorders the input sequences based on their lengths (hlens) in descending order, - and filters out invalid examples where the input length is smaller than the minimum required length. - - Raises: - Warning: If all examples in the batch are invalid for Bayes Risk CTC calculation. - - Example: - >>> brctc = BayesRiskCTC() - >>> nnet_output = torch.randn(32, 100, 50) # (batch_size, max_time, num_classes) - >>> ys_pad = torch.randint(0, 50, (32, 20)) # (batch_size, max_target_length) - >>> hlens = torch.randint(80, 101, (32,)) # Input lengths - >>> ylens = torch.randint(15, 21, (32,)) # Target lengths - >>> loss = brctc(nnet_output, ys_pad, hlens, ylens) - """ # Reorder and filter out invalid examples: # A. K2 requires that hlens are in descending order; # B. remove all examples whose hlens is smaller than necessary. @@ -111,38 +47,6 @@ def forward(self, nnet_output, ys_pad, hlens, ylens): return loss_utt def forward_core(self, nnet_output, ys, hlens, ylens): - """ - Compute the core Bayes Risk CTC loss for a batch of sequences. - - This method implements the main logic of the Bayes Risk CTC loss calculation, - including building the CTC graphs, performing intersections, and computing - forward-backward scores. - - Args: - nnet_output (torch.Tensor): Output from the neural network, shape (B, T, C), - where B is batch size, T is the maximum sequence length, and C is the number of classes. - ys (List[List[int]]): List of target label sequences (not padded). - hlens (torch.Tensor): Lengths of input sequences, shape (B,). - ylens (torch.Tensor): Lengths of target sequences, shape (B,). - - Returns: - torch.Tensor: The computed loss for each sequence in the batch, shape (B,). - - Note: - This method uses the K2 library for finite-state automata operations and - implements the core algorithm of Bayes Risk CTC as described in the paper. - - Raises: - NotImplementedError: If an unsupported group_strategy is specified. - - Example: - >>> brctc = BayesRiskCTC() - >>> nnet_output = torch.randn(32, 100, 50) # (batch_size, max_time, num_classes) - >>> ys = [[1, 2, 3], [4, 5, 6, 7], ...] # List of target sequences - >>> hlens = torch.randint(80, 101, (32,)) # Input lengths - >>> ylens = torch.tensor([len(y) for y in ys]) # Target lengths - >>> loss = brctc.forward_core(nnet_output, ys, hlens, ylens) - """ # (1) Find the shape (B, T, _), U = nnet_output.size(), max(ylens) @@ -243,36 +147,7 @@ def forward_core(self, nnet_output, ys, hlens, ylens): raise NotImplementedError def get_risk_scores(self, loss_state, hlens, risk_factor): - """ - Calculate the Bayes risk scores for each state in the lattice. - - This method computes risk scores based on the specified risk strategy, - which are then used to modify the loss calculation in the Bayes Risk CTC algorithm. - - Args: - loss_state (torch.Tensor): The loss state tensor, shape (B, U, T), - where B is batch size, U is the number of unique labels, - and T is the maximum sequence length. - hlens (torch.Tensor): Lengths of input sequences, shape (B,). - risk_factor (float): The risk factor to scale the computed risk. - - Returns: - torch.Tensor: The computed risk scores, shape (B, U, T). - - Raises: - NotImplementedError: If an unsupported risk_strategy is specified. - - Note: - The risk calculation depends on the risk_strategy attribute: - - 'exp': Exponential risk based on the position in the sequence. - - 'exp_rel': Exponential risk relative to the position of maximum loss. - - Example: - >>> brctc = BayesRiskCTC(risk_strategy='exp', risk_factor=0.1) - >>> loss_state = torch.randn(32, 20, 100) # (batch_size, num_labels, max_time) - >>> hlens = torch.randint(80, 101, (32,)) # Input lengths - >>> risk_scores = brctc.get_risk_scores(loss_state, hlens, 0.1) - """ + """Add the bayes risk in multiple ways""" B, U, T = loss_state.size() if self.risk_strategy == "exp": @@ -302,40 +177,6 @@ def get_risk_scores(self, loss_state, hlens, risk_factor): def find_all_index( self, ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, arc_map_b ): - """ - Find indices for arcs and states in the lattice. - - This method computes various indices for arcs and states in the CTC lattice, - which are used in the forward-backward algorithm of the Bayes Risk CTC loss calculation. - - Args: - ragged_lat (k2.RaggedTensor): The ragged lattice tensor. - ctc_graph (k2.Fsa): The CTC graph. - dense_fsa_vec (k2.DenseFsaVec): The dense FSA vector. - arc_map_a (torch.Tensor): Arc map for the first FSA. - arc_map_b (torch.Tensor): Arc map for the second FSA. - - Returns: - Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]]: - A tuple containing two tuples: - 1. Arc indices: (arc_u_idx, arc_t_idx, arc_k_idx, arc_b_idx) - 2. State indices: (state_u_idx, state_t_idx, state_k_idx, state_b_idx) - - Where: - - u_idx: label indices - - t_idx: time indices - - k_idx: class indices - - b_idx: batch indices - - Note: - This method handles the indexing for both arcs and states in the lattice, - including special handling for start and end states. - - Example: - >>> brctc = BayesRiskCTC() - >>> # Assume ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, and arc_map_b are properly initialized - >>> arc_indices, state_indices = brctc.find_all_index(ragged_lat, ctc_graph, dense_fsa_vec, arc_map_a, arc_map_b) - """ # This function finds the index of (b, t, u, d) for each arc and each state num_fsas = len(ctc_graph.arcs.row_splits(1)) - 1 @@ -398,37 +239,6 @@ def find_all_index( ) def find_minimum_hlens(self, ys_pad, ylens): - """ - Calculate the minimum possible input lengths for each sequence in the batch. - - This method computes the minimum required input length for each target sequence, - considering the CTC alignment rules. It also removes padding from the target sequences. - - Args: - ys_pad (torch.Tensor): Padded target sequences, shape (B, U), - where B is batch size and U is the maximum target sequence length. - ylens (torch.Tensor): Lengths of target sequences, shape (B,). - - Returns: - Tuple[List[List[int]], torch.Tensor]: - - List[List[int]]: Unpadded target sequences. - - torch.Tensor: Minimum required input lengths for each sequence, shape (B,). - - Note: - The minimum input length for a CTC alignment is calculated by considering - that each label requires at least one frame, and consecutive identical - labels require an additional frame between them. - - Example: - >>> brctc = BayesRiskCTC() - >>> ys_pad = torch.tensor([[1, 2, 2, 3, 0], [4, 4, 5, 0, 0]]) - >>> ylens = torch.tensor([4, 3]) - >>> ys, min_hlens = brctc.find_minimum_hlens(ys_pad, ylens) - >>> print(ys) - [[1, 2, 2, 3], [4, 4, 5]] - >>> print(min_hlens) - tensor([5, 4]) - """ device = ys_pad.device ys_pad, ylens = ys_pad.cpu().tolist(), ylens.cpu().tolist() ys, min_hlens = [], [] @@ -455,31 +265,7 @@ def find_minimum_hlens(self, ys_pad, ylens): def log_substraction_exp(a, b): - """ - Compute log(exp(a) - exp(b)) in a numerically stable way. - - This function calculates log(exp(a) - exp(b)) using a numerically stable - implementation to avoid overflow and underflow issues that can occur when - dealing with very large or very small exponents. - - Args: - a (torch.Tensor): The first input tensor. - b (torch.Tensor): The second input tensor. Should have the same shape as 'a'. - - Returns: - torch.Tensor: The result of log(exp(a) - exp(b)), computed element-wise. - - Note: - This function handles cases where 'a' or 'b' contain infinite values. - - Examples: - >>> import torch - >>> a = torch.tensor([1.0, 2.0, 3.0]) - >>> b = torch.tensor([0.5, 1.5, 2.5]) - >>> result = log_substraction_exp(a, b) - >>> print(result) - tensor([0.9401, 1.5501, 2.5501]) - """ + """return (a.exp() - b.exp()).log(): a numeracal safe implementation""" ans = torch.ones_like(a) * float("-inf") # avoid -inf in input diff --git a/espnet2/asr/ctc.py b/espnet2/asr/ctc.py index 5365ee90b0d..3c61cbee242 100644 --- a/espnet2/asr/ctc.py +++ b/espnet2/asr/ctc.py @@ -7,47 +7,16 @@ class CTC(torch.nn.Module): - """ - CTC (Connectionist Temporal Classification) module. - - This class implements various CTC loss functions for sequence-to-sequence models, - particularly useful in speech recognition tasks. - - Attributes: - ctc_lo (torch.nn.Linear): Linear layer for CTC output. - ctc_loss (callable): CTC loss function based on the specified type. - dropout_rate (float): Dropout rate applied to the input. - ctc_type (str): Type of CTC loss to use ('builtin', 'builtin2', 'gtnctc', or 'brctc'). - reduce (bool): Whether to reduce the CTC loss into a scalar. + """CTC module. Args: - odim (int): Dimension of outputs. - encoder_output_size (int): Number of encoder projection units. - dropout_rate (float, optional): Dropout rate (0.0 ~ 1.0). Defaults to 0.0. - ctc_type (str, optional): Type of CTC loss. Defaults to "builtin". - reduce (bool, optional): Whether to reduce the CTC loss. Defaults to True. - ignore_nan_grad (bool, optional): Ignore NaN gradients (deprecated, use zero_infinity). - zero_infinity (bool, optional): Zero infinite losses and associated gradients. Defaults to True. - brctc_risk_strategy (str, optional): Risk strategy for Bayes Risk CTC. Defaults to "exp". - brctc_group_strategy (str, optional): Group strategy for Bayes Risk CTC. Defaults to "end". - brctc_risk_factor (float, optional): Risk factor for Bayes Risk CTC. Defaults to 0.0. - - Raises: - ValueError: If ctc_type is not one of "builtin", "gtnctc", or "brctc". - ImportError: If K2 is not installed when using Bayes Risk CTC. - - Note: - The class supports different CTC implementations, including the built-in PyTorch CTC, - GTN-based CTC, and Bayes Risk CTC. The choice of CTC type affects the behavior and - performance of the loss calculation. - - Example: - >>> ctc = CTC(odim=1000, encoder_output_size=256, ctc_type="builtin") - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) - >>> hlens = torch.full((32,), 100) # (batch_size,) - >>> ys_pad = torch.randint(0, 1000, (32, 50)) # (batch_size, max_label_length) - >>> ys_lens = torch.randint(10, 50, (32,)) # (batch_size,) - >>> loss = ctc(hs_pad, hlens, ys_pad, ys_lens) + odim: dimension of outputs + encoder_output_size: number of encoder projection units + dropout_rate: dropout rate (0.0 ~ 1.0) + ctc_type: builtin or gtnctc + reduce: reduce the CTC loss into a scalar + ignore_nan_grad: Same as zero_infinity (keeping for backward compatiblity) + zero_infinity: Whether to zero infinite losses and the associated gradients. """ @typechecked @@ -104,41 +73,6 @@ def __init__( self.reduce = reduce def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: - """ - Calculate the CTC loss based on the specified CTC type. - - This method computes the CTC loss using the predefined CTC loss function, - which varies depending on the CTC type specified during initialization. - - Args: - th_pred (torch.Tensor): Predicted probabilities or logits. - Shape: (batch_size, max_time, num_classes) - th_target (torch.Tensor): Target labels. - Shape: (sum(target_lengths)) - th_ilen (torch.Tensor): Input lengths. - Shape: (batch_size,) - th_olen (torch.Tensor): Output lengths. - Shape: (batch_size,) - - Returns: - torch.Tensor: Computed CTC loss. - - Raises: - NotImplementedError: If an unsupported CTC type is specified. - - Note: - - For 'builtin' and 'brctc' types, the input is expected to be log probabilities. - - For 'builtin2', NaN gradients are handled differently based on the 'ignore_nan_grad' flag. - - For 'gtnctc', the input is converted to log probabilities within the method. - - Example: - >>> ctc = CTC(odim=10, encoder_output_size=20, ctc_type="builtin") - >>> pred = torch.randn(2, 5, 10) # (batch_size, max_time, num_classes) - >>> target = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) # (sum(target_lengths)) - >>> input_length = torch.tensor([5, 5]) # (batch_size,) - >>> target_length = torch.tensor([3, 5]) # (batch_size,) - >>> loss = ctc.loss_fn(pred, target, input_length, target_length) - """ if self.ctc_type == "builtin" or self.ctc_type == "brctc": th_pred = th_pred.log_softmax(2).float() loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) @@ -217,38 +151,13 @@ def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: raise NotImplementedError def forward(self, hs_pad, hlens, ys_pad, ys_lens): - """ - Calculate CTC loss for the input sequences. - - This method applies the CTC loss calculation to the input hidden state sequences. - It first applies a linear transformation and dropout to the input, then computes - the CTC loss based on the specified CTC type. + """Calculate CTC loss. Args: - hs_pad (torch.Tensor): Batch of padded hidden state sequences. - Shape: (batch_size, max_time, hidden_size) - hlens (torch.Tensor): Batch of lengths of hidden state sequences. - Shape: (batch_size,) - ys_pad (torch.Tensor): Batch of padded character id sequence tensor. - Shape: (batch_size, max_label_length) - ys_lens (torch.Tensor): Batch of lengths of character sequences. - Shape: (batch_size,) - - Returns: - torch.Tensor: Computed CTC loss. - - Note: - - The method handles different CTC types ('brctc', 'gtnctc', and others) differently. - - For 'gtnctc', the target sequences are converted to a list format. - - For other types, the target sequences are flattened into a 1D tensor. - - Example: - >>> ctc = CTC(odim=1000, encoder_output_size=256) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) - >>> hlens = torch.full((32,), 100) # (batch_size,) - >>> ys_pad = torch.randint(0, 1000, (32, 50)) # (batch_size, max_label_length) - >>> ys_lens = torch.randint(10, 50, (32,)) # (batch_size,) - >>> loss = ctc(hs_pad, hlens, ys_pad, ys_lens) + hs_pad: batch of padded hidden state sequences (B, Tmax, D) + hlens: batch of lengths of hidden state sequences (B) + ys_pad: batch of padded character id sequence tensor (B, Lmax) + ys_lens: batch of lengths of character sequence (B) """ # hs_pad: (B, L, NProj) -> ys_hat: (B, L, Nvocab) ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate)) @@ -275,74 +184,31 @@ def forward(self, hs_pad, hlens, ys_pad, ys_lens): return loss def softmax(self, hs_pad): - """ - Apply softmax to frame activations. - - This method applies a linear transformation followed by softmax to the input - hidden state sequences, typically used for obtaining output probabilities. + """softmax of frame activations Args: - hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. - Shape: (batch_size, max_time, hidden_size) - + Tensor hs_pad: 3d tensor (B, Tmax, eprojs) Returns: - torch.Tensor: Softmax applied 3D tensor. - Shape: (batch_size, max_time, output_dim) - - Example: - >>> ctc = CTC(odim=1000, encoder_output_size=256) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) - >>> softmax_output = ctc.softmax(hs_pad) - >>> softmax_output.shape - torch.Size([32, 100, 1000]) + torch.Tensor: softmax applied 3d tensor (B, Tmax, odim) """ return F.softmax(self.ctc_lo(hs_pad), dim=2) def log_softmax(self, hs_pad): - """ - Apply log softmax to frame activations. - - This method applies a linear transformation followed by log softmax to the input - hidden state sequences, typically used for obtaining log probabilities. + """log_softmax of frame activations Args: - hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. - Shape: (batch_size, max_time, hidden_size) - + Tensor hs_pad: 3d tensor (B, Tmax, eprojs) Returns: - torch.Tensor: Log softmax applied 3D tensor. - Shape: (batch_size, max_time, output_dim) - - Example: - >>> ctc = CTC(odim=1000, encoder_output_size=256) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) - >>> log_softmax_output = ctc.log_softmax(hs_pad) - >>> log_softmax_output.shape - torch.Size([32, 100, 1000]) + torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim) """ return F.log_softmax(self.ctc_lo(hs_pad), dim=2) def argmax(self, hs_pad): - """ - Apply argmax to frame activations. - - This method applies a linear transformation followed by argmax to the input - hidden state sequences, typically used for obtaining the most likely class - for each time step. + """argmax of frame activations Args: - hs_pad (torch.Tensor): 3D tensor of padded hidden state sequences. - Shape: (batch_size, max_time, hidden_size) - + torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs) Returns: - torch.Tensor: Argmax applied 2D tensor. - Shape: (batch_size, max_time) - - Example: - >>> ctc = CTC(odim=1000, encoder_output_size=256) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, hidden_size) - >>> argmax_output = ctc.argmax(hs_pad) - >>> argmax_output.shape - torch.Size([32, 100]) + torch.Tensor: argmax applied 2d tensor (B, Tmax) """ return torch.argmax(self.ctc_lo(hs_pad), dim=2) diff --git a/espnet2/asr/decoder/abs_decoder.py b/espnet2/asr/decoder/abs_decoder.py index 6497c94024b..e46d1c24fcb 100644 --- a/espnet2/asr/decoder/abs_decoder.py +++ b/espnet2/asr/decoder/abs_decoder.py @@ -7,30 +7,6 @@ class AbsDecoder(torch.nn.Module, ScorerInterface, ABC): - """ - Abstract base class for decoders in speech recognition models. - - This class defines the interface for decoders used in the ESPnet framework. - It inherits from torch.nn.Module for neural network functionality, - ScorerInterface for scoring methods, and ABC for abstract base class behavior. - - Attributes: - None - - Note: - Subclasses must implement the `forward` method. - - Examples: - >>> class MyDecoder(AbsDecoder): - ... def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens): - ... # Implement decoder logic here - ... pass - ... - ... def score(self, ys, state, x): - ... # Implement scoring logic here - ... pass - """ - @abstractmethod def forward( self, @@ -39,39 +15,4 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the decoder. - - This abstract method defines the forward pass for the decoder. It takes - encoded features and target sequences as input and produces decoded output. - - Args: - hs_pad (torch.Tensor): Padded hidden state sequences from the encoder. - Shape: (batch, time, hidden_dim) - hlens (torch.Tensor): Lengths of hidden state sequences. - Shape: (batch,) - ys_in_pad (torch.Tensor): Padded input token sequences for teacher forcing. - Shape: (batch, output_length) - ys_in_lens (torch.Tensor): Lengths of input token sequences. - Shape: (batch,) - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - Decoded output sequences. Shape: (batch, output_length, vocab_size) - - Attention weights or None. Shape: (batch, output_length, input_length) - - Raises: - NotImplementedError: This method must be implemented by subclasses. - - Note: - This method should be implemented by all subclasses of AbsDecoder. - - Examples: - >>> class MyDecoder(AbsDecoder): - ... def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens): - ... # Decoder implementation - ... decoded_output = ... # Shape: (batch, output_length, vocab_size) - ... attention_weights = ... # Shape: (batch, output_length, input_length) - ... return decoded_output, attention_weights - """ raise NotImplementedError diff --git a/espnet2/asr/decoder/hugging_face_transformers_decoder.py b/espnet2/asr/decoder/hugging_face_transformers_decoder.py index 464af8217a0..1af31b3679d 100644 --- a/espnet2/asr/decoder/hugging_face_transformers_decoder.py +++ b/espnet2/asr/decoder/hugging_face_transformers_decoder.py @@ -27,39 +27,11 @@ class HuggingFaceTransformersDecoder(AbsDecoder, BatchScorerInterface): - """ - A decoder class that utilizes Hugging Face Transformers models. - - This class implements a decoder interface using pre-trained models from the - Hugging Face Transformers library. It supports both causal language models - and sequence-to-sequence models. - - Attributes: - causal_lm (bool): Whether the model is a causal language model. - decoder (torch.nn.Module): The main decoder component from the Hugging Face model. - decoder_word_embeddings (torch.nn.Embedding): Word embeddings of the decoder. - decoder_pad_token_id (int): Padding token ID for the decoder. - tokenizer_padding_side (str): Padding side used by the tokenizer ('left' or 'right'). - prefix (torch.Tensor): Embedded prefix tokens. - postfix (torch.Tensor): Embedded postfix tokens. - lm_head (torch.nn.Module): Language model head for output projection. - model_name_or_path (str): Name or path of the Hugging Face model. - linear_in (torch.nn.Module): Linear layer to match encoder output size to decoder input size. + """Hugging Face Transformers Decoder. Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimension of the encoder output. - model_name_or_path (str): Name or path of the Hugging Face Transformers model. - causal_lm (bool, optional): Whether to use a causal language model. Defaults to False. - prefix (str, optional): Prefix to add to the input. Defaults to "". - postfix (str, optional): Postfix to add to the input. Defaults to "". - - Raises: - ImportError: If the 'transformers' library is not installed. - - Note: - This class requires the 'transformers' library to be installed. - It adapts Hugging Face models for use with the ESPnet framework. + encoder_output_size: dimension of encoder attention + model_name_or_path: Hugging Face Transformers model name """ @typechecked @@ -145,26 +117,19 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the decoder. - - This method processes the encoder output and generates decoder output scores. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of the encoded sequences in the batch. - ys_in_pad (torch.Tensor): Input tensor of shape (batch, maxlen_out, #mels). - ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch. - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: input tensor (batch, maxlen_out, #mels) + ys_in_lens: (batch) Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - x (torch.Tensor): Decoded token scores before softmax, - of shape (batch, maxlen_out, token). - - ys_in_lens (torch.Tensor): Lengths of the input sequences. - - Note: - The method handles both causal language models and sequence-to-sequence models. - For causal language models, it adds prefix and postfix to the input. + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ enc_out = self.linear_in(hs_pad) @@ -219,18 +184,7 @@ def forward( return x, ys_in_lens def reload_pretrained_parameters(self): - """ - Reloads the pretrained parameters for the decoder and language model head. - - This method restores the original pretrained weights of the Hugging Face - Transformers model. It resets both the decoder and the language model head - to their initial pretrained states. - - Note: - This method is useful for resetting the model to its pretrained state, - which can be beneficial in certain training or fine-tuning scenarios. - It logs a message upon successful reloading of the parameters. - """ + self.decoder.load_state_dict(self.decoder_pretrained_params) if self.lm_head_pretrained_params is not None: self.lm_head.load_state_dict(self.lm_head_pretrained_params) @@ -238,33 +192,6 @@ def reload_pretrained_parameters(self): logging.info("Pretrained Transformers model parameters reloaded!") def add_prefix_postfix(self, enc_out, hlens, ys_in_pad, ys_in_lens): - """ - Adds prefix and postfix to the encoder output for causal language models. - - This method prepares the input for causal language models by adding - prefix and postfix embeddings to the encoder output. It also handles - padding and creates the appropriate attention mask. - - Args: - enc_out (torch.Tensor): Encoder output tensor. - hlens (torch.Tensor): Lengths of the encoder outputs in the batch. - ys_in_pad (torch.Tensor): Input tensor (batch, maxlen_out, #mels). - ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch. - - Returns: - Tuple[Dict[str, torch.Tensor], torch.Tensor]: - - args (Dict[str, torch.Tensor]): A dictionary containing: - - 'inputs_embeds': Tensor with added prefix, postfix, and padding. - - 'attention_mask': Attention mask for the inputs. - - 'return_dict': Boolean set to True. - - no_loss_lengths (torch.Tensor): Lengths of the sequences excluding - the parts that should not contribute to the loss calculation. - - Note: - This method is specifically designed for use with causal language models - in the HuggingFaceTransformersDecoder class. It handles different padding - sides based on the tokenizer configuration. - """ args = {} hlens_max = (hlens + ys_in_lens).max() @@ -309,28 +236,6 @@ def add_prefix_postfix(self, enc_out, hlens, ys_in_pad, ys_in_lens): return args, no_loss_lengths def score(self, ys, state, x, speech=None): - """ - Computes the score for the next token given the current state and input. - - This method generates the log probability scores for the next token in the sequence - using the Hugging Face model's generation process. - - Args: - ys (torch.Tensor): Current token sequence. - state (Any): The current state (not used in this implementation). - x (torch.Tensor): The encoder output. - speech (torch.Tensor, optional): Speech input (not used in this implementation). - - Returns: - Tuple[torch.Tensor, None]: - - next_token_scores (torch.Tensor): Log probability scores for the next token. - - None: Placeholder for the updated state (not used in this implementation). - - Note: - This method is part of the BatchScorerInterface and is used in beam search - decoding. It prepares the input for the Hugging Face model, runs the forward - pass, and returns the log probabilities for the next token. - """ model_kwargs = { "encoder_outputs": ModelOutput( last_hidden_state=self.linear_in(x).unsqueeze(0) @@ -359,30 +264,6 @@ def batch_score( xs: torch.Tensor, speech: torch.Tensor = None, ) -> Tuple[torch.Tensor, List[Any]]: - """ - Computes scores for the next tokens in a batch of sequences. - - This method generates log probability scores for the next tokens in multiple - sequences simultaneously using the Hugging Face model's generation process. - - Args: - ys (torch.Tensor): Batch of current token sequences. - states (List[Any]): List of current states (not used in this implementation). - xs (torch.Tensor): Batch of encoder outputs. - speech (torch.Tensor, optional): Batch of speech inputs (not used in this implementation). - - Returns: - Tuple[torch.Tensor, None]: - - next_token_scores (torch.Tensor): Log probability scores for the next tokens - in the batch. Shape: (batch_size, vocab_size). - - None: Placeholder for the updated states (not used in this implementation). - - Note: - This method is part of the BatchScorerInterface and is used for efficient - batch processing in beam search decoding. It prepares the input for the - Hugging Face model, runs the forward pass, and returns the log probabilities - for the next tokens in all sequences of the batch. - """ # import pdb;pdb.set_trace() model_kwargs = { "encoder_outputs": ModelOutput(last_hidden_state=self.linear_in(xs)), @@ -404,33 +285,6 @@ def batch_score( def get_hugging_face_model_network(model): - """ - Retrieves the network component from a Hugging Face model. - - This function attempts to extract the main network component from various - Hugging Face model architectures. It checks for common attribute names - used in different model implementations. - - Args: - model: A Hugging Face model object. - - Returns: - The network component of the model. - - Raises: - Exception: If the network attribute cannot be found in the model. - - Examples: - >>> from transformers import AutoModelForCausalLM - >>> model = AutoModelForCausalLM.from_pretrained("gpt2") - >>> network = get_hugging_face_model_network(model) - >>> print(type(network).__name__) - 'GPT2Model' - - Note: - This function supports models with 'transformer', 'gpt_neox', or 'model' - attributes. If none of these attributes are found, an exception is raised. - """ if hasattr(model, "transformer"): network = model.transformer elif hasattr(model, "gpt_neox"): @@ -444,33 +298,6 @@ def get_hugging_face_model_network(model): def get_hugging_face_model_lm_head(model): - """ - Retrieves the language model head component from a Hugging Face model. - - This function attempts to extract the language model head component from - various Hugging Face model architectures. It checks for common attribute - names used in different model implementations. - - Args: - model: A Hugging Face model object. - - Returns: - The language model head component of the model. - - Raises: - Exception: If the LM head attribute cannot be found in the model. - - Examples: - >>> from transformers import AutoModelForCausalLM - >>> model = AutoModelForCausalLM.from_pretrained("gpt2") - >>> lm_head = get_hugging_face_model_lm_head(model) - >>> print(type(lm_head).__name__) - 'Linear' - - Note: - This function supports models with 'lm_head' or 'embed_out' attributes. - If neither of these attributes are found, an exception is raised. - """ if hasattr(model, "lm_head"): lm_head = model.lm_head elif hasattr(model, "embed_out"): diff --git a/espnet2/asr/decoder/mlm_decoder.py b/espnet2/asr/decoder/mlm_decoder.py index 344f39301a8..a787185de11 100644 --- a/espnet2/asr/decoder/mlm_decoder.py +++ b/espnet2/asr/decoder/mlm_decoder.py @@ -20,50 +20,6 @@ class MLMDecoder(AbsDecoder): - """ - Masked Language Model (MLM) Decoder for sequence-to-sequence models. - - This class implements a decoder for Masked Language Modeling tasks, typically - used in transformer-based architectures. It processes encoded input sequences - and generates output sequences with the ability to handle masked tokens. - - Attributes: - embed (torch.nn.Sequential): Input embedding layer. - normalize_before (bool): Whether to apply layer normalization before each decoder block. - after_norm (LayerNorm): Layer normalization applied after all decoder blocks if normalize_before is True. - output_layer (torch.nn.Linear or None): Linear layer for final output projection. - decoders (torch.nn.ModuleList): List of decoder layers. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in position-wise feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer, either "embed" or "linear". Defaults to "embed". - use_output_layer (bool, optional): Whether to use an output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to apply layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - - Note: - This decoder adds an extra token to the vocabulary size to account for the mask token. - - Example: - >>> encoder_output_size = 256 - >>> vocab_size = 1000 - >>> decoder = MLMDecoder(vocab_size, encoder_output_size) - >>> hs_pad = torch.randn(32, 50, encoder_output_size) # (batch, max_len, feat) - >>> hlens = torch.full((32,), 50, dtype=torch.long) - >>> ys_in_pad = torch.randint(0, vocab_size, (32, 30)) # (batch, max_len) - >>> ys_in_lens = torch.full((32,), 30, dtype=torch.long) - >>> decoded, olens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) - """ - @typechecked def __init__( self, @@ -134,43 +90,21 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the MLM Decoder. - - This method processes the input sequences through the decoder layers, - applying self-attention, source attention, and feed-forward operations. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). - ys_in_pad (torch.Tensor): Input token ids, int64 tensor of shape (batch, maxlen_out) - if input_layer is "embed", or input tensor of shape (batch, maxlen_out, #mels) - for other input layer types. - ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) Returns: - tuple: A tuple containing: - - x (torch.Tensor): Decoded token scores before softmax, shape (batch, maxlen_out, token) - if use_output_layer is True. If use_output_layer is False, returns the - last decoder layer's output. - - olens (torch.Tensor): Output sequence lengths, shape (batch,). - - Raises: - ValueError: If the input shapes are inconsistent with the expected dimensions. - - Example: - >>> decoder = MLMDecoder(1000, 256) - >>> hs_pad = torch.randn(32, 50, 256) - >>> hlens = torch.full((32,), 50, dtype=torch.long) - >>> ys_in_pad = torch.randint(0, 1000, (32, 30)) - >>> ys_in_lens = torch.full((32,), 30, dtype=torch.long) - >>> decoded, olens = decoder.forward(hs_pad, hlens, ys_in_pad, ys_in_lens) - >>> print(decoded.shape, olens.shape) - torch.Size([32, 30, 1001]) torch.Size([32]) - - Note: - The method handles masking for both the target and memory sequences to ensure - proper attention mechanisms during decoding. + (tuple): tuple containing: + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ tgt = ys_in_pad # tgt_mask: (B, 1, L) diff --git a/espnet2/asr/decoder/rnn_decoder.py b/espnet2/asr/decoder/rnn_decoder.py index 61738f8f212..634108357a2 100644 --- a/espnet2/asr/decoder/rnn_decoder.py +++ b/espnet2/asr/decoder/rnn_decoder.py @@ -30,50 +30,6 @@ def build_attention_list( han_conv_filts: int = 100, han_win: int = 5, ): - """ - Build a list of attention modules based on the specified parameters. - - This function creates and returns a list of attention modules for use in - speech recognition models. It supports both single and multi-encoder - configurations, as well as hierarchical attention networks (HAN). - - Args: - eprojs (int): Number of encoder projection units. - dunits (int): Number of decoder units. - atype (str, optional): Attention type. Defaults to "location". - num_att (int, optional): Number of attention modules. Defaults to 1. - num_encs (int, optional): Number of encoders. Defaults to 1. - aheads (int, optional): Number of attention heads. Defaults to 4. - adim (int, optional): Attention dimension. Defaults to 320. - awin (int, optional): Attention window size. Defaults to 5. - aconv_chans (int, optional): Number of attention convolution channels. Defaults to 10. - aconv_filts (int, optional): Number of attention convolution filters. Defaults to 100. - han_mode (bool, optional): Whether to use hierarchical attention network. Defaults to False. - han_type (str, optional): Type of hierarchical attention. Defaults to None. - han_heads (int, optional): Number of HAN attention heads. Defaults to 4. - han_dim (int, optional): HAN attention dimension. Defaults to 320. - han_conv_chans (int, optional): Number of HAN convolution channels. Defaults to -1. - han_conv_filts (int, optional): Number of HAN convolution filters. Defaults to 100. - han_win (int, optional): HAN window size. Defaults to 5. - - Returns: - torch.nn.ModuleList: A list of attention modules. - - Raises: - ValueError: If the number of encoders is less than one. - - Note: - The function behavior changes based on the number of encoders and whether - hierarchical attention network mode is enabled. - - Examples: - >>> att_list = build_attention_list(256, 320, num_encs=2, han_mode=True) - >>> print(len(att_list)) - 1 - >>> att_list = build_attention_list(256, 320, num_encs=2, han_mode=False) - >>> print(len(att_list)) - 2 - """ att_list = torch.nn.ModuleList() if num_encs == 1: for i in range(num_att): @@ -124,54 +80,6 @@ def build_attention_list( class RNNDecoder(AbsDecoder): - """ - RNN-based decoder for sequence-to-sequence models. - - This class implements a recurrent neural network (RNN) decoder, which can be - used in various sequence-to-sequence tasks such as speech recognition or - machine translation. It supports both LSTM and GRU cell types, multiple - layers, and various attention mechanisms. - - The decoder uses an embedding layer, followed by multiple RNN layers with - dropout, and an output layer. It also incorporates attention mechanisms to - focus on different parts of the input sequence during decoding. - - Attributes: - embed (torch.nn.Embedding): Embedding layer for input tokens. - decoder (torch.nn.ModuleList): List of RNN cells (LSTM or GRU). - dropout_dec (torch.nn.ModuleList): List of dropout layers for RNN outputs. - output (torch.nn.Linear): Output layer for vocabulary distribution. - att_list (torch.nn.ModuleList): List of attention modules. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - rnn_type (str, optional): Type of RNN cell to use ('lstm' or 'gru'). Defaults to "lstm". - num_layers (int, optional): Number of RNN layers. Defaults to 1. - hidden_size (int, optional): Hidden size of the RNN. Defaults to 320. - sampling_probability (float, optional): Probability of sampling from previous output. Defaults to 0.0. - dropout (float, optional): Dropout probability. Defaults to 0.0. - context_residual (bool, optional): Whether to use context residual connection. Defaults to False. - replace_sos (bool, optional): Whether to replace token. Defaults to False. - num_encs (int, optional): Number of encoders. Defaults to 1. - att_conf (dict, optional): Configuration for attention modules. Defaults to get_default_kwargs(build_attention_list). - - Raises: - ValueError: If an unsupported RNN type is specified. - - Note: - This decoder supports both single and multi-encoder configurations, as well as - speaker parallel attention (SPA) for multi-speaker scenarios. - - Example: - >>> decoder = RNNDecoder(vocab_size=1000, encoder_output_size=256, num_layers=2, hidden_size=512) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, encoder_output_size) - >>> hlens = torch.full((32,), 100) # (batch_size,) - >>> ys_in_pad = torch.randint(0, 1000, (32, 20)) # (batch_size, max_output_length) - >>> ys_in_lens = torch.full((32,), 20) # (batch_size,) - >>> decoder_output, output_lengths = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) - """ - @typechecked def __init__( self, @@ -242,65 +150,9 @@ def __init__( ) def zero_state(self, hs_pad): - """ - Initialize a zero state for the decoder. - - This method creates a tensor of zeros with the same batch size as the input - and the decoder's hidden size. It's typically used to initialize the hidden - state of the RNN at the start of the decoding process. - - Args: - hs_pad (torch.Tensor): The padded hidden state tensor from the encoder. - It is used to determine the batch size for the zero state. - - Returns: - torch.Tensor: A tensor of zeros with shape (batch_size, hidden_size), - where batch_size is inferred from hs_pad and hidden_size is the - decoder's hidden size (self.dunits). - - Example: - >>> decoder = RNNDecoder(...) - >>> hs_pad = torch.randn(32, 100, 256) # (batch_size, max_time, encoder_output_size) - >>> initial_state = decoder.zero_state(hs_pad) - >>> print(initial_state.shape) - torch.Size([32, 320]) # Assuming self.dunits = 320 - """ return hs_pad.new_zeros(hs_pad.size(0), self.dunits) def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): - """ - Perform a forward pass through the RNN layers. - - This method processes the input through all layers of the RNN (LSTM or GRU), - applying dropout between vertical connections. - - Args: - ey (torch.Tensor): Input tensor, typically the concatenation of the - embedded previous output and the context vector. - z_list (list): List to store output hidden states for each layer. - c_list (list): List to store cell states for each layer (used for LSTM only). - z_prev (list): List of previous hidden states for each layer. - c_prev (list): List of previous cell states for each layer (used for LSTM only). - - Returns: - tuple: A tuple containing: - - z_list (list): Updated list of output hidden states for each layer. - - c_list (list): Updated list of cell states for each layer (for LSTM only). - - Note: - - For LSTM, both hidden state (z) and cell state (c) are updated. - - For GRU, only the hidden state (z) is updated. - - Dropout is applied to the output of each layer except the last one. - - Example: - >>> decoder = RNNDecoder(...) - >>> ey = torch.randn(32, 512) # (batch_size, embed_size + context_size) - >>> z_list = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] - >>> c_list = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] - >>> z_prev = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] - >>> c_prev = [torch.zeros(32, 320) for _ in range(decoder.dlayers)] - >>> z_list, c_list = decoder.rnn_forward(ey, z_list, c_list, z_prev, c_prev) - """ if self.dtype == "lstm": z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) for i in range(1, self.dlayers): @@ -317,47 +169,6 @@ def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): return z_list, c_list def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): - """ - Perform a forward pass of the RNN decoder. - - This method processes the encoder outputs and generates decoder outputs - for the entire sequence. - - Args: - hs_pad (torch.Tensor or List[torch.Tensor]): Padded hidden states from encoder(s). - For single encoder: tensor of shape (batch, time, hidden_size). - For multiple encoders: list of such tensors. - hlens (torch.Tensor or List[torch.Tensor]): Lengths of encoder hidden states. - For single encoder: tensor of shape (batch,). - For multiple encoders: list of such tensors. - ys_in_pad (torch.Tensor): Padded input label sequences. - Shape: (batch, sequence_length). - ys_in_lens (torch.Tensor): Lengths of input label sequences. - Shape: (batch,). - strm_idx (int, optional): Stream index for multi-speaker attention. - Defaults to 0. - - Returns: - tuple: A tuple containing: - - z_all (torch.Tensor): Output sequence scores. - Shape: (batch, sequence_length, vocab_size). - - ys_in_lens (torch.Tensor): Lengths of input sequences. - - Note: - - This method supports both single and multiple encoder scenarios. - - It implements teacher forcing with optional scheduled sampling. - - For multiple encoders, hierarchical attention is used. - - Example: - >>> decoder = RNNDecoder(...) - >>> hs_pad = torch.randn(32, 100, 256) # (batch, time, hidden_size) - >>> hlens = torch.full((32,), 100) - >>> ys_in_pad = torch.randint(0, 1000, (32, 20)) # (batch, sequence_length) - >>> ys_in_lens = torch.full((32,), 20) - >>> z_all, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) - >>> print(z_all.shape) - torch.Size([32, 20, 1000]) # (batch, sequence_length, vocab_size) - """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: @@ -442,41 +253,6 @@ def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): return z_all, ys_in_lens def init_state(self, x): - """ - Initialize the decoder state for inference. - - This method sets up the initial state of the decoder, including hidden states, - cell states (for LSTM), and attention weights. It's typically used at the start - of the decoding process during inference. - - Args: - x (torch.Tensor or List[torch.Tensor]): The encoder output(s). - For single encoder: tensor of shape (batch, time, hidden_size). - For multiple encoders: list of such tensors. - - Returns: - dict: A dictionary containing the initial decoder state with keys: - - c_prev (list): Initial cell states for each layer (for LSTM). - - z_prev (list): Initial hidden states for each layer. - - a_prev (None or list): Initial attention weights. - - workspace (tuple): Additional workspace information including: - - att_idx (int): Index of the current attention module. - - z_list (list): List of initial hidden states. - - c_list (list): List of initial cell states (for LSTM). - - Note: - - The method handles both single and multiple encoder scenarios. - - For multiple encoders, it initializes states for all encoders and the - hierarchical attention network (HAN). - - The attention modules are reset to clear any pre-computed values. - - Example: - >>> decoder = RNNDecoder(...) - >>> x = torch.randn(1, 100, 256) # (batch, time, hidden_size) - >>> initial_state = decoder.init_state(x) - >>> print(initial_state.keys()) - dict_keys(['c_prev', 'z_prev', 'a_prev', 'workspace']) - """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: @@ -506,44 +282,6 @@ def init_state(self, x): ) def score(self, yseq, state, x): - """ - Calculate the log probability score for the next token. - - This method computes the score for the next token in the sequence given the - current state and encoder outputs. It's typically used in beam search decoding. - - Args: - yseq (torch.Tensor): Current output sequence. - Shape: (sequence_length,). - state (dict): Current decoder state containing: - - c_prev (list): Previous cell states for each layer (for LSTM). - - z_prev (list): Previous hidden states for each layer. - - a_prev (None or list): Previous attention weights. - - workspace (tuple): Additional workspace information. - x (torch.Tensor or List[torch.Tensor]): The encoder output(s). - For single encoder: tensor of shape (batch, time, hidden_size). - For multiple encoders: list of such tensors. - - Returns: - tuple: A tuple containing: - - logp (torch.Tensor): Log probability scores for the next token. - Shape: (vocab_size,). - - new_state (dict): Updated decoder state. - - Note: - - This method supports both single and multiple encoder scenarios. - - For multiple encoders, it uses hierarchical attention. - - The context vector is concatenated with the embedded input for scoring. - - Example: - >>> decoder = RNNDecoder(...) - >>> yseq = torch.tensor([1, 2, 3]) # Current sequence - >>> x = torch.randn(1, 100, 256) # (batch, time, hidden_size) - >>> state = decoder.init_state(x) - >>> logp, new_state = decoder.score(yseq, state, x) - >>> print(logp.shape) - torch.Size([1000]) # Assuming vocab_size = 1000 - """ # to support mutiple encoder asr mode, in single encoder mode, # convert torch.Tensor to List of torch.Tensor if self.num_encs == 1: diff --git a/espnet2/asr/decoder/s4_decoder.py b/espnet2/asr/decoder/s4_decoder.py index cfa02217267..30828bcb23d 100644 --- a/espnet2/asr/decoder/s4_decoder.py +++ b/espnet2/asr/decoder/s4_decoder.py @@ -12,43 +12,25 @@ class S4Decoder(AbsDecoder, BatchScorerInterface): - """ - S4 decoder module for sequence-to-sequence models. - - This class implements a decoder based on the S4 (Structured State Space Sequence) model. - It can be used in various sequence-to-sequence tasks, such as automatic speech recognition. + """S4 decoder module. Args: - vocab_size (int): Size of the vocabulary (output dimension). - encoder_output_size (int): Dimension of the encoder's hidden vector. - input_layer (str, optional): Type of input layer. Defaults to "embed". - dropinp (float, optional): Input dropout rate. Defaults to 0.0. - dropout (float, optional): Dropout rate applied to every residual and layer. Defaults to 0.25. - prenorm (bool, optional): If True, use pre-normalization; otherwise, post-normalization. Defaults to True. - n_layers (int, optional): Number of layers in the decoder. Defaults to 16. - transposed (bool, optional): If True, transpose inputs so each layer receives (batch, dim, length). Defaults to False. - tie_dropout (bool, optional): If True, tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d. Defaults to False. - n_repeat (int, optional): Number of times each layer is repeated per stage before applying pooling. Defaults to 1. - layer (dict, optional): Layer configuration. Must be specified. - residual (dict, optional): Residual connection configuration. - norm (dict, optional): Normalization configuration (e.g., layer vs batch). - pool (dict, optional): Configuration for pooling layer per stage. - track_norms (bool, optional): If True, log norms of each layer output. Defaults to True. - drop_path (float, optional): Drop rate for stochastic depth. Defaults to 0.0. - - Attributes: - d_model (int): Dimension of the model (same as encoder_output_size). - sos (int): Start-of-sequence token ID. - eos (int): End-of-sequence token ID. - odim (int): Output dimension (same as vocab_size). - dropout (float): Dropout rate. - embed (torch.nn.Embedding): Embedding layer for input tokens. - dropout_emb (torch.nn.Dropout): Dropout layer for embeddings. - decoder (SequenceModel): Main S4 decoder model. - output (torch.nn.Linear): Output linear layer. - - Note: - This decoder implements the BatchScorerInterface, allowing for efficient batch scoring of hypotheses. + vocab_size: output dim + encoder_output_size: dimension of hidden vector + input_layer: input layer type + dropinp: input dropout + dropout: dropout parameter applied on every residual and every layer + prenorm: pre-norm vs. post-norm + n_layers: number of layers + transposed: transpose inputs so each layer receives (batch, dim, length) + tie_dropout: tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d + n_repeat: each layer is repeated n times per stage before applying pooling + layer: layer config, must be specified + residual: residual config + norm: normalization config (e.g. layer vs batch) + pool: config for pooling layer per stage + track_norms: log norms of each layer output + drop_path: drop rate for stochastic depth """ @typechecked @@ -105,27 +87,7 @@ def __init__( self.output = torch.nn.Linear(self.d_model, vocab_size) def init_state(self, x: torch.Tensor): - """ - Initialize the decoder state. - - This method initializes the state of the S4 decoder. It creates a default state - for the decoder based on the input tensor's device. - - Args: - x (torch.Tensor): An input tensor used to determine the device for state initialization. - - Returns: - Any: The initialized state of the decoder. - - Note: - The state is initialized with a batch size of 1, regardless of the input tensor's batch size. - This method is typically called before starting the decoding process. - - Example: - >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) - >>> x = torch.randn(1, 10, 512) # (batch_size, sequence_length, feature_dim) - >>> initial_state = decoder.init_state(x) - """ + """Initialize state.""" return self.decoder.default_state(1, device=x.device) def forward( @@ -136,39 +98,22 @@ def forward( ys_in_lens: torch.Tensor, state=None, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the S4 decoder. - - This method performs the forward pass of the S4 decoder, processing the encoder output - and the input token sequence to generate decoded token scores. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, float32 tensor of shape (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of the encoded sequences in the batch, shape (batch,). - ys_in_pad (torch.Tensor): Input token ids, int64 tensor of shape (batch, maxlen_out). - If input_layer is "embed", this contains token ids. - Otherwise, it contains input tensors of shape (batch, maxlen_out, #mels). - ys_in_lens (torch.Tensor): Lengths of the input sequences in the batch, shape (batch,). - state (Optional[Any]): Initial state for the decoder. Defaults to None. - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - decoded (torch.Tensor): Decoded token scores before softmax, - shape (batch, maxlen_out, vocab_size). - - ys_in_lens (torch.Tensor): Lengths of the input sequences, shape (batch,). + (tuple): tuple containing: - Note: - This method applies the embedding layer, processes the sequence through the S4 decoder, - and applies the output layer to generate token scores. - - Example: - >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) - >>> hs_pad = torch.randn(2, 100, 512) # (batch_size, max_encoder_length, encoder_dim) - >>> hlens = torch.tensor([100, 80]) - >>> ys_in_pad = torch.randint(0, 1000, (2, 20)) # (batch_size, max_decoder_length) - >>> ys_in_lens = torch.tensor([20, 15]) - >>> decoded, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) - >>> print(decoded.shape) # (2, 20, 1000) + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ memory = hs_pad memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( @@ -188,61 +133,24 @@ def forward( return decoded, ys_in_lens def score(self, ys, state, x): - """ - Score a sequence of tokens. - - This method is not implemented for the S4Decoder. - - Args: - ys: The sequence of tokens to be scored. - state: The current state of the decoder. - x: The input features. - - Raises: - NotImplementedError: This method is not implemented for S4Decoder. - - Note: - This method is part of the BatchScorerInterface, but it is not implemented - for the S4Decoder. Use the `batch_score` method instead for efficient - batch scoring of hypotheses. - """ raise NotImplementedError def batch_score( self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor ) -> Tuple[torch.Tensor, List[Any]]: - """ - Score new token batches efficiently. - - This method computes scores for the next token given a batch of prefix tokens - and their corresponding states. It is part of the BatchScorerInterface and is - used for efficient batch decoding. + """Score new token batch. Args: - ys (torch.Tensor): Prefix tokens of shape (n_batch, ylen). - Contains int64 token IDs. - states (List[Any]): List of scorer states for prefix tokens. - Each state corresponds to a sequence in the batch. - xs (torch.Tensor): The encoder features that generate ys. - Shape (n_batch, xlen, n_feat). + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). Returns: - Tuple[torch.Tensor, List[Any]]: A tuple containing: - - logp (torch.Tensor): Log probabilities for the next token. - Shape (n_batch, n_vocab). - - states_list (List[Any]): Updated states for each sequence in the batch. - - Note: - This method is designed for use in beam search decoding, where multiple - hypotheses are scored simultaneously. + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. - Example: - >>> decoder = S4Decoder(vocab_size=1000, encoder_output_size=512) - >>> ys = torch.randint(0, 1000, (5, 10)) # (n_batch, ylen) - >>> states = [decoder.init_state(torch.randn(1, 512)) for _ in range(5)] - >>> xs = torch.randn(5, 100, 512) # (n_batch, xlen, n_feat) - >>> logp, new_states = decoder.batch_score(ys, states, xs) - >>> print(logp.shape) # (5, 1000) """ # merge states n_batch = len(ys) diff --git a/espnet2/asr/decoder/transducer_decoder.py b/espnet2/asr/decoder/transducer_decoder.py index 0d198ecc2e5..857a7d09215 100644 --- a/espnet2/asr/decoder/transducer_decoder.py +++ b/espnet2/asr/decoder/transducer_decoder.py @@ -10,48 +10,17 @@ class TransducerDecoder(AbsDecoder): - """ - (RNN-)Transducer decoder module. - - This class implements a decoder for (RNN-)Transducer models, which can be used - in automatic speech recognition (ASR) systems. It supports both LSTM and GRU - as the underlying recurrent neural network types. - - Attributes: - embed: Embedding layer for input tokens. - dropout_embed: Dropout layer applied to the embedding output. - decoder: List of RNN layers (LSTM or GRU). - dropout_dec: List of dropout layers applied after each RNN layer. - dlayers: Number of decoder layers. - dunits: Number of hidden units in each decoder layer. - dtype: Type of RNN used ('lstm' or 'gru'). - odim: Output dimension (vocabulary size). - ignore_id: ID to ignore in the input (default: -1). - blank_id: ID representing the blank/pad token. - device: Device on which the model is allocated. + """(RNN-)Transducer decoder module. Args: - vocab_size: Size of the vocabulary (output dimension). - rnn_type: Type of RNN to use ('lstm' or 'gru'). Defaults to 'lstm'. - num_layers: Number of decoder layers. Defaults to 1. - hidden_size: Number of hidden units in each decoder layer. Defaults to 320. - dropout: Dropout rate for decoder layers. Defaults to 0.0. - dropout_embed: Dropout rate for the embedding layer. Defaults to 0.0. - embed_pad: ID of the padding token in the embedding layer. Defaults to 0. - - Raises: - ValueError: If an unsupported RNN type is specified. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> labels = torch.randint(0, 1000, (32, 10)) # Batch of 32, sequence length 10 - >>> output = decoder(labels) - >>> print(output.shape) - torch.Size([32, 10, 320]) # (batch_size, sequence_length, hidden_size) - - Note: - This implementation follows the Google Python Style Guide and PEP 8 standards. - The decoder can be used as part of a larger (RNN-)Transducer model for ASR tasks. + vocab_size: Output dimension. + layers_type: (RNN-)Decoder layers type. + num_layers: Number of decoder layers. + hidden_size: Number of decoder units per layer. + dropout: Dropout rate for decoder layers. + dropout_embed: Dropout rate for embedding layer. + embed_pad: Embed/Blank symbol ID. + """ @typechecked @@ -97,56 +66,25 @@ def __init__( self.device = next(self.parameters()).device def set_device(self, device: torch.device): - """ - Set GPU device to use. - - This method sets the device (GPU) on which the decoder will operate. + """Set GPU device to use. Args: - device (torch.device): The PyTorch device object representing the GPU - to be used for computations. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000) - >>> device = torch.device("cuda:0") - >>> decoder.set_device(device) - - Note: - This method should be called before performing any computations if you want - to explicitly set the GPU device. If not called, the decoder will use the - device of its parameters by default. + device: Device ID. + """ self.device = device def init_state( self, batch_size: int ) -> Tuple[torch.Tensor, Optional[torch.tensor]]: - """ - Initialize decoder states. - - This method initializes the hidden states of the decoder for a given batch size. - It creates zero tensors for both LSTM (hidden state and cell state) and GRU (hidden state only). + """Initialize decoder states. Args: - batch_size (int): The batch size for which to initialize the states. + batch_size: Batch size. Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - h_n (torch.Tensor): The initial hidden state tensor of shape (N, B, D_dec), - where N is the number of layers, B is the batch size, and D_dec is the - decoder's hidden size. - - c_n (Optional[torch.Tensor]): The initial cell state tensor for LSTM, - with the same shape as h_n. For GRU, this will be None. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> batch_size = 32 - >>> h_n, c_n = decoder.init_state(batch_size) - >>> print(h_n.shape, c_n.shape) - torch.Size([2, 32, 320]) torch.Size([2, 32, 320]) - - Note: - The initialized states are created on the same device as the decoder. + : Initial decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + """ h_n = torch.zeros( self.dlayers, @@ -172,38 +110,16 @@ def rnn_forward( sequence: torch.Tensor, state: Tuple[torch.Tensor, Optional[torch.Tensor]], ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """ - Encode source label sequences through the RNN layers. - - This method passes the input sequence through all RNN layers of the decoder, - applying dropout after each layer. + """Encode source label sequences. Args: - sequence (torch.Tensor): RNN input sequences of shape (B, D_emb), - where B is the batch size and D_emb is the embedding dimension. - state (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. - For LSTM, it's a tuple of (hidden_state, cell_state), each of shape (N, B, D_dec). - For GRU, it's a tuple of (hidden_state, None). - N is the number of layers, B is the batch size, and D_dec is the decoder's hidden size. + sequence: RNN input sequences. (B, D_emb) + state: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) Returns: - Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - - sequence (torch.Tensor): RNN output sequences of shape (B, D_dec). - - (h_next, c_next) (Tuple[torch.Tensor, Optional[torch.Tensor]]): - Updated decoder hidden states. For LSTM, both h_next and c_next - have shape (N, B, D_dec). For GRU, c_next is None. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> batch_size, seq_len, hidden_size = 32, 10, 320 - >>> input_seq = torch.randn(batch_size, hidden_size) - >>> initial_state = decoder.init_state(batch_size) - >>> output_seq, (h_next, c_next) = decoder.rnn_forward(input_seq, initial_state) - >>> print(output_seq.shape, h_next.shape, c_next.shape) - torch.Size([32, 320]) torch.Size([2, 32, 320]) torch.Size([2, 32, 320]) - - Note: - This method handles both LSTM and GRU architectures based on the decoder's configuration. + sequence: RNN output sequences. (B, D_dec) + (h_next, c_next): Decoder hidden states. (N, B, D_dec), (N, B, D_dec)) + """ h_prev, c_prev = state h_next, c_next = self.init_state(sequence.size(0)) @@ -229,33 +145,14 @@ def rnn_forward( return sequence, (h_next, c_next) def forward(self, labels: torch.Tensor) -> torch.Tensor: - """ - Encode source label sequences. - - This method processes input label sequences through the decoder's embedding layer - and RNN layers to produce decoder output sequences. + """Encode source label sequences. Args: - labels (torch.Tensor): Label ID sequences of shape (B, L), where B is the - batch size and L is the sequence length. + labels: Label ID sequences. (B, L) Returns: - torch.Tensor: Decoder output sequences of shape (B, T, U, D_dec), where - T is the number of time steps, U is the number of label tokens, and - D_dec is the decoder's hidden size. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> batch_size, seq_length = 32, 10 - >>> labels = torch.randint(0, 1000, (batch_size, seq_length)) - >>> output = decoder(labels) - >>> print(output.shape) - torch.Size([32, 10, 320]) - - Note: - This method applies dropout to the embedded input before passing it through - the RNN layers. The output shape might differ from the example if the decoder - is configured differently. + dec_out: Decoder output sequences. (B, T, U, D_dec) + """ init_state = self.init_state(labels.size(0)) dec_embed = self.dropout_embed(self.embed(labels)) @@ -267,35 +164,17 @@ def forward(self, labels: torch.Tensor) -> torch.Tensor: def score( self, hyp: Hypothesis, cache: Dict[str, Any] ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]], torch.Tensor]: - """ - Perform one-step forward computation for a single hypothesis. - - This method computes the decoder output for the next step given a current hypothesis. - It uses a cache to store and retrieve previously computed results for efficiency. + """One-step forward hypothesis. Args: - hyp (Hypothesis): The current hypothesis containing the label sequence and decoder state. - cache (Dict[str, Any]): A dictionary storing pairs of (dec_out, state) for each label sequence. + hyp: Hypothesis. + cache: Pairs of (dec_out, state) for each label sequence. (key) Returns: - Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]], torch.Tensor]: - - dec_out (torch.Tensor): Decoder output sequence of shape (1, D_dec). - - new_state (Tuple[torch.Tensor, Optional[torch.Tensor]]): Updated decoder hidden states. - For LSTM, it's (hidden_state, cell_state), each of shape (N, 1, D_dec). - For GRU, it's (hidden_state, None). - - label (torch.Tensor): Label ID for language model, shape (1,). - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000) - >>> hyp = Hypothesis(...) # Create a hypothesis - >>> cache = {} - >>> dec_out, new_state, label = decoder.score(hyp, cache) - >>> print(dec_out.shape, label.shape) - torch.Size([1, 320]) torch.Size([1]) - - Note: - This method is typically used in beam search decoding for Transducer models. - The cache helps avoid redundant computations for previously seen label sequences. + dec_out: Decoder output sequence. (1, D_dec) + new_state: Decoder hidden states. ((N, 1, D_dec), (N, 1, D_dec)) + label: Label ID for LM. (1,) + """ label = torch.full((1, 1), hyp.yseq[-1], dtype=torch.long, device=self.device) @@ -318,40 +197,19 @@ def batch_score( cache: Dict[str, Any], use_lm: bool, ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: - """ - Perform one-step forward computation for a batch of hypotheses. - - This method computes the decoder output for the next step given a batch of current hypotheses. - It uses a cache to store and retrieve previously computed results for efficiency. + """One-step forward hypotheses. Args: - hyps (Union[List[Hypothesis], List[ExtendedHypothesis]]): A list of current hypotheses. - dec_states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. - For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). - For GRU, it's (hidden_state, None). - cache (Dict[str, Any]): A dictionary storing pairs of (dec_out, dec_states) for each label sequence. - use_lm (bool): Whether to compute label ID sequences for language model integration. + hyps: Hypotheses. + states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + cache: Pairs of (dec_out, dec_states) for each label sequences. (keys) + use_lm: Whether to compute label ID sequences for LM. Returns: - Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: - - dec_out (torch.Tensor): Decoder output sequences of shape (B, D_dec). - - dec_states (Tuple[torch.Tensor, torch.Tensor]): Updated decoder hidden states. - For LSTM, both elements have shape (N, B, D_dec). For GRU, the second element is None. - - lm_labels (torch.Tensor): Label ID sequences for language model, shape (B,). - Returns None if use_lm is False. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000) - >>> hyps = [Hypothesis(...) for _ in range(5)] # Batch of 5 hypotheses - >>> dec_states = decoder.init_state(5) - >>> cache = {} - >>> dec_out, new_states, lm_labels = decoder.batch_score(hyps, dec_states, cache, use_lm=True) - >>> print(dec_out.shape, lm_labels.shape) - torch.Size([5, 320]) torch.Size([5, 1]) - - Note: - This method is typically used in batch beam search decoding for Transducer models. - It's more efficient than scoring hypotheses individually, especially for larger batch sizes. + dec_out: Decoder output sequences. (B, D_dec) + dec_states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + lm_labels: Label ID sequences for LM. (B,) + """ final_batch = len(hyps) @@ -400,35 +258,16 @@ def batch_score( def select_state( self, states: Tuple[torch.Tensor, Optional[torch.Tensor]], idx: int ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """ - Extract a specific state from the decoder hidden states. - - This method selects and returns the hidden state for a given index from the batch - of decoder hidden states. + """Get specified ID state from decoder hidden states. Args: - states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Decoder hidden states. - For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). - For GRU, it's (hidden_state, None). - N is the number of layers, B is the batch size, and D_dec is the decoder's hidden size. - idx (int): The index of the state to extract. + states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + idx: State ID to extract. Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: The selected decoder hidden state. - - For LSTM: (hidden_state, cell_state), each of shape (N, 1, D_dec). - - For GRU: (hidden_state, None), where hidden_state has shape (N, 1, D_dec). - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> batch_size = 32 - >>> states = decoder.init_state(batch_size) - >>> selected_state = decoder.select_state(states, 5) - >>> print(selected_state[0].shape, selected_state[1].shape) - torch.Size([2, 1, 320]) torch.Size([2, 1, 320]) - - Note: - This method is useful when you need to extract the state for a specific - hypothesis from a batch of states, typically during beam search decoding. + : Decoder hidden state for given ID. + ((N, 1, D_dec), (N, 1, D_dec)) + """ return ( states[0][:, idx : idx + 1, :], @@ -441,40 +280,15 @@ def create_batch_states( new_states: List[Tuple[torch.Tensor, Optional[torch.Tensor]]], check_list: Optional[List] = None, ) -> List[Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """ - Create batch decoder hidden states from individual states. - - This method combines individual decoder states into a batch, which is useful - for parallel processing of multiple hypotheses. + """Create decoder hidden states. Args: - states (Tuple[torch.Tensor, Optional[torch.Tensor]]): Initial decoder hidden states. - For LSTM, it's (hidden_state, cell_state), each of shape (N, B, D_dec). - For GRU, it's (hidden_state, None). - new_states (List[Tuple[torch.Tensor, Optional[torch.Tensor]]]): List of individual - decoder hidden states to be combined. - Each element is of shape (N, 1, D_dec) for LSTM hidden and cell states, - or (N, 1, D_dec) and None for GRU. - check_list (Optional[List]): Not used in the current implementation. - Kept for potential future use or compatibility. + states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + new_states: Decoder hidden states. [N x ((1, D_dec), (1, D_dec))] Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: Combined batch of decoder hidden states. - - For LSTM: (hidden_state, cell_state), each of shape (N, B, D_dec). - - For GRU: (hidden_state, None), where hidden_state has shape (N, B, D_dec). - N is the number of layers, B is the new batch size (length of new_states), - and D_dec is the decoder's hidden size. - - Example: - >>> decoder = TransducerDecoder(vocab_size=1000, rnn_type='lstm', num_layers=2) - >>> individual_states = [decoder.init_state(1) for _ in range(5)] - >>> batch_states = decoder.create_batch_states(decoder.init_state(5), individual_states) - >>> print(batch_states[0].shape, batch_states[1].shape) - torch.Size([2, 5, 320]) torch.Size([2, 5, 320]) - - Note: - This method is particularly useful in beam search decoding when combining - states from different hypotheses into a single batch for efficient processing. + states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec)) + """ return ( torch.cat([s[0] for s in new_states], dim=1), diff --git a/espnet2/asr/decoder/transformer_decoder.py b/espnet2/asr/decoder/transformer_decoder.py index 66320b03d10..7abb4baad14 100644 --- a/espnet2/asr/decoder/transformer_decoder.py +++ b/espnet2/asr/decoder/transformer_decoder.py @@ -29,32 +29,25 @@ class BaseTransformerDecoder(AbsDecoder, BatchScorerInterface): - """ - Base class for Transformer decoder modules. - - This abstract base class provides the foundation for implementing various - Transformer decoder architectures. It defines the common structure and - methods that all Transformer decoder variants should implement. - - Attributes: - embed (torch.nn.Sequential): The embedding layer for input tokens. - decoders (torch.nn.ModuleList): List of decoder layers (to be implemented by subclasses). - after_norm (LayerNorm): Layer normalization applied after the decoder stack. - output_layer (torch.nn.Linear): Linear layer for final output projection. + """Base class of Transfomer decoder module. Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - dropout_rate (float): Dropout rate. - positional_dropout_rate (float): Dropout rate for positional encoding. - input_layer (str): Type of input layer ('embed' or 'linear'). - use_output_layer (bool): Whether to use an output layer. - pos_enc_class: Positional encoding class to use. - normalize_before (bool): Whether to apply layer normalization before each block. - - Note: - Subclasses should implement the specific decoder architecture by - defining the `decoders` attribute. + vocab_size: output dim + encoder_output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of decoder blocks + dropout_rate: dropout rate + self_attention_dropout_rate: dropout rate for attention + input_layer: input layer type + use_output_layer: whether to use output layer + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) """ @typechecked @@ -110,33 +103,25 @@ def forward( return_hs: bool = False, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the decoder. - - This method processes the encoder output and generates decoded sequences. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). - ys_in_pad (torch.Tensor): Input token ids or features, shape (batch, maxlen_out). - If input_layer is "embed", it contains token ids. - Otherwise, it contains input features. - ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). - return_hs (bool, optional): Whether to return the last hidden state. Defaults to False. - return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + return_hs: (bool) whether to return the last hidden output + before output layer + return_all_hs: (bool) whether to return all the hidden intermediates Returns: - tuple: A tuple containing: - - x (torch.Tensor): Decoded token scores before softmax, - shape (batch, maxlen_out, vocab_size) if use_output_layer is True. - - olens (torch.Tensor): Output lengths, shape (batch,). - - hidden (torch.Tensor, optional): Last hidden state if return_hs is True. - - intermediate_outs (List[torch.Tensor], optional): All intermediate hidden states - if return_all_hs is True. - - Note: - The behavior of this method can be customized using the return_hs and - return_all_hs flags to obtain additional hidden state information. + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ tgt = ys_in_pad # tgt_mask: (B, 1, L) @@ -189,29 +174,21 @@ def forward_one_step( cache: List[torch.Tensor] = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """ - Perform one step of the decoder forward pass. - - This method is typically used for incremental decoding, processing one token at a time. + """Forward one step. Args: - tgt (torch.Tensor): Input token ids, shape (batch, maxlen_out). - tgt_mask (torch.Tensor): Input token mask, shape (batch, maxlen_out). - dtype=torch.uint8 in PyTorch 1.2-, dtype=torch.bool in PyTorch 1.2+ (including 1.2). - memory (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). - memory_mask (torch.Tensor, optional): Encoded memory mask, shape (batch, 1, maxlen_in). - cache (List[torch.Tensor], optional): Cached output list of shape (batch, max_time_out-1, size). - return_hs (bool, optional): Whether to return the hidden state. Defaults to False. - + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + memory_mask: encoded memory mask (batch, 1, maxlen_in) + cache: cached output list of (batch, max_time_out-1, size) + return_hs: dec hidden state corresponding to ys, + used for searchable hidden ints Returns: - tuple: A tuple containing: - - y (torch.Tensor): Output tensor of shape (batch, maxlen_out, token) - if use_output_layer is True, else the last hidden state. - - new_cache (List[torch.Tensor]): Updated cache for each decoder layer. - - hidden (torch.Tensor, optional): Hidden state if return_hs is True. - - Note: - This method is crucial for efficient autoregressive decoding in inference time. + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) """ x = self.embed(tgt) if cache is None: @@ -237,28 +214,7 @@ def forward_one_step( return y, new_cache def score(self, ys, state, x, return_hs=False): - """ - Score a sequence of tokens. - - This method computes the log probability score for a given sequence of tokens. - - Args: - ys (torch.Tensor): Sequence of tokens to score, shape (sequence_length,). - state (List[Any]): Previous decoder state. - x (torch.Tensor): Encoder output, shape (1, encoder_output_size). - return_hs (bool, optional): Whether to return the hidden state. Defaults to False. - - Returns: - tuple: A tuple containing: - - logp (torch.Tensor): Log probability scores for the input sequence, - shape (sequence_length, vocab_size). - - state (List[Any]): Updated decoder state. - - hs (torch.Tensor, optional): Hidden state if return_hs is True. - - Note: - This method is typically used in beam search and other decoding algorithms - to evaluate and rank candidate sequences. - """ + """Score.""" ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) if return_hs: (logp, hs), state = self.forward_one_step( @@ -286,27 +242,20 @@ def batch_score( xs: torch.Tensor, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[Any]]: - """ - Score a batch of token sequences. - - This method computes the log probability scores for a batch of token sequences. + """Score new token batch. Args: - ys (torch.Tensor): Batch of token sequences, shape (batch_size, sequence_length). - states (List[Any]): List of scorer states for prefix tokens. - xs (torch.Tensor): Batch of encoder outputs, shape (batch_size, max_length, encoder_output_size). - return_hs (bool, optional): Whether to return the hidden states. Defaults to False. + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + Returns: - tuple: A tuple containing: - - logp (torch.Tensor): Batch of log probability scores for the next token, - shape (batch_size, vocab_size). - - state_list (List[List[Any]]): Updated list of scorer states for each sequence in the batch. - - hs (torch.Tensor, optional): Hidden states if return_hs is True. - - Note: - This method is optimized for batch processing, which is more efficient than - scoring sequences individually, especially during beam search or batch decoding. + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + """ # merge states n_batch = len(ys) @@ -418,40 +367,6 @@ def batch_score_partially_AR( class TransformerDecoder(BaseTransformerDecoder): - """ - Transformer decoder module. - - This class implements the standard Transformer decoder architecture as described - in "Attention Is All You Need" (Vaswani et al., 2017). It consists of multiple - stacked self-attention and encoder-decoder attention layers, followed by a - feed-forward network. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - layer_drop_rate (float, optional): Layer dropout rate. Defaults to 0.0. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers. - - Note: - This implementation allows for easy modification of various hyperparameters - and architectural choices, making it suitable for a wide range of sequence - generation tasks. - """ - @typechecked def __init__( self, @@ -503,41 +418,6 @@ def __init__( class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder): - """ - Lightweight Convolution Transformer decoder module. - - This class implements a Transformer decoder that replaces the self-attention - mechanism with lightweight convolution, as described in "Pay Less Attention - with Lightweight and Dynamic Convolutions" (Wu et al., 2019). It combines - the benefits of convolutional neural networks and self-attention. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - conv_wshare (int, optional): Weight sharing factor for convolution. Defaults to 4. - conv_kernel_length (Sequence[int], optional): Kernel size for each convolution layer. Defaults to (11, 11, 11, 11, 11, 11). - conv_usebias (bool, optional): Whether to use bias in convolution layers. Defaults to False. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers with lightweight convolution. - - Note: - This decoder variant can be more efficient than standard self-attention - for certain tasks, especially those involving long sequences. - """ - @typechecked def __init__( self, @@ -600,41 +480,6 @@ def __init__( class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder): - """ - Lightweight 2D Convolution Transformer decoder module. - - This class implements a Transformer decoder that uses 2D lightweight convolutions - instead of self-attention. It extends the concept introduced in "Pay Less Attention - with Lightweight and Dynamic Convolutions" (Wu et al., 2019) to 2D convolutions, - potentially capturing more complex patterns in the input. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - conv_wshare (int, optional): Weight sharing factor for 2D convolution. Defaults to 4. - conv_kernel_length (Sequence[int], optional): Kernel size for each 2D convolution layer. Defaults to (11, 11, 11, 11, 11, 11). - conv_usebias (bool, optional): Whether to use bias in 2D convolution layers. Defaults to False. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers with 2D lightweight convolution. - - Note: - This decoder variant may be particularly effective for tasks where the input has - a 2D structure or where capturing 2D patterns is beneficial. - """ - @typechecked def __init__( self, @@ -697,43 +542,6 @@ def __init__( class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder): - """ - Dynamic Convolution Transformer decoder module. - - This class implements a Transformer decoder that replaces the self-attention - mechanism with dynamic convolution, as introduced in "Pay Less Attention with - Lightweight and Dynamic Convolutions" (Wu et al., 2019). Dynamic convolution - adapts its weights based on the input, allowing for more flexible and - context-dependent processing. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - conv_wshare (int, optional): Weight sharing factor for dynamic convolution. Defaults to 4. - conv_kernel_length (Sequence[int], optional): Kernel size for each dynamic convolution layer. Defaults to (11, 11, 11, 11, 11, 11). - conv_usebias (bool, optional): Whether to use bias in dynamic convolution layers. Defaults to False. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers with dynamic convolution. - - Note: - Dynamic convolution can be more effective than standard self-attention or - lightweight convolution for certain tasks, especially those requiring - adaptive processing of input sequences. - """ - @typechecked def __init__( self, @@ -796,43 +604,6 @@ def __init__( class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder): - """ - Dynamic 2D Convolution Transformer decoder module. - - This class implements a Transformer decoder that uses 2D dynamic convolutions - instead of self-attention. It extends the concept of dynamic convolutions - introduced in "Pay Less Attention with Lightweight and Dynamic Convolutions" - (Wu et al., 2019) to 2D, allowing for more complex and adaptive processing - of input sequences with potential 2D structure. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - conv_wshare (int, optional): Weight sharing factor for 2D dynamic convolution. Defaults to 4. - conv_kernel_length (Sequence[int], optional): Kernel size for each 2D dynamic convolution layer. Defaults to (11, 11, 11, 11, 11, 11). - conv_usebias (bool, optional): Whether to use bias in 2D dynamic convolution layers. Defaults to False. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers with 2D dynamic convolution. - - Note: - This decoder variant may be particularly effective for tasks where the input - has a 2D structure or where capturing adaptive 2D patterns is beneficial. - It combines the flexibility of dynamic convolutions with 2D processing capabilities. - """ - @typechecked def __init__( self, @@ -895,40 +666,6 @@ def __init__( class TransformerMDDecoder(BaseTransformerDecoder): - """ - Transformer Multi-Decoder (MD) module. - - This class implements a Transformer decoder with an additional attention mechanism - for speech input, making it suitable for multi-modal tasks such as speech translation - or speech recognition with auxiliary text input. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Dimensionality of the encoder output. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of decoder layers. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - self_attention_dropout_rate (float, optional): Dropout rate for self-attention. Defaults to 0.0. - src_attention_dropout_rate (float, optional): Dropout rate for source attention. Defaults to 0.0. - input_layer (str, optional): Type of input layer ('embed' or 'linear'). Defaults to "embed". - use_output_layer (bool, optional): Whether to use output layer. Defaults to True. - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before each block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - use_speech_attn (bool, optional): Whether to use additional attention for speech input. Defaults to True. - - Attributes: - decoders (torch.nn.ModuleList): List of decoder layers with additional speech attention mechanism. - use_speech_attn (bool): Indicates whether speech attention is used. - - Note: - This decoder is designed for tasks that involve both text and speech inputs, - allowing for more effective integration of multi-modal information. The additional - speech attention mechanism can be toggled on or off based on the task requirements. - """ - @typechecked def __init__( self, @@ -996,34 +733,24 @@ def forward( speech_lens: torch.Tensor = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the TransformerMDDecoder. - - This method processes the encoder output, speech input (if applicable), and generates - decoded sequences, optionally returning intermediate hidden states. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of encoded sequences, shape (batch,). - ys_in_pad (torch.Tensor): Input token ids or features, shape (batch, maxlen_out). - If input_layer is "embed", it contains token ids. - Otherwise, it contains input features. - ys_in_lens (torch.Tensor): Lengths of input sequences, shape (batch,). - speech (torch.Tensor, optional): Speech input tensor, shape (batch, speech_maxlen, speech_feat). - speech_lens (torch.Tensor, optional): Lengths of speech input sequences, shape (batch,). - return_hs (bool, optional): Whether to return the last hidden state. Defaults to False. - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + return_hs: dec hidden state corresponding to ys, + used for searchable hidden ints Returns: - tuple: A tuple containing: - - x (torch.Tensor): Decoded token scores before softmax, - shape (batch, maxlen_out, vocab_size) if use_output_layer is True. - - olens (torch.Tensor): Output lengths, shape (batch,). - - hs_asr (torch.Tensor, optional): Last hidden state if return_hs is True. - - Note: - This method supports multi-modal decoding by incorporating both text and speech - inputs when available. The speech attention mechanism is used only if - use_speech_attn is True and speech input is provided. + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ tgt = ys_in_pad # tgt_mask: (B, 1, L) @@ -1080,34 +807,23 @@ def forward_one_step( cache: List[torch.Tensor] = None, return_hs: bool = False, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """ - Perform one step of the decoder forward pass. - - This method is designed for incremental decoding, processing one token at a time - and optionally incorporating speech input. + """Forward one step. Args: - tgt (torch.Tensor): Input token ids, shape (batch, maxlen_out). - tgt_mask (torch.Tensor): Input token mask, shape (batch, maxlen_out). - dtype=torch.uint8 in PyTorch 1.2-, dtype=torch.bool in PyTorch 1.2+ (including 1.2). - memory (torch.Tensor): Encoded memory, shape (batch, maxlen_in, feat). - memory_mask (torch.Tensor, optional): Encoded memory mask, shape (batch, 1, maxlen_in). - speech (torch.Tensor, optional): Speech input tensor, shape (batch, speech_maxlen, speech_feat). - speech_mask (torch.Tensor, optional): Speech input mask, shape (batch, 1, speech_maxlen). - cache (List[torch.Tensor], optional): Cached output list of shape (batch, max_time_out-1, size). - return_hs (bool, optional): Whether to return the hidden state. Defaults to False. - + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + memory_mask: encoded memory mask (batch, 1, maxlen_in) + speech: encoded speech, float32 (batch, maxlen_in, feat) + speech_mask: encoded memory mask (batch, 1, maxlen_in) + cache: cached output list of (batch, max_time_out-1, size) + return_hs: dec hidden state corresponding to ys, + used for searchable hidden ints Returns: - tuple: A tuple containing: - - y (torch.Tensor): Output tensor of shape (batch, maxlen_out, token) - if use_output_layer is True, else the last hidden state. - - h_asr (torch.Tensor, optional): Hidden state if return_hs is True. - - new_cache (List[torch.Tensor]): Updated cache for each decoder layer. - - Note: - This method supports multi-modal decoding by incorporating both text and speech - inputs when available. The speech attention mechanism is used only if - use_speech_attn is True and speech input is provided. + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) """ x = self.embed(tgt) if cache is None: @@ -1146,30 +862,7 @@ def forward_one_step( return y, new_cache def score(self, ys, state, x, speech=None): - """ - Score a sequence of tokens. - - This method computes the log probability score for a given sequence of tokens, - optionally incorporating speech input. - - Args: - ys (torch.Tensor): Sequence of tokens to score, shape (sequence_length,). - state (List[Any]): Previous decoder state. - x (torch.Tensor): Encoder output, shape (1, encoder_output_size). - speech (torch.Tensor, optional): Speech input tensor, shape (1, speech_maxlen, speech_feat). - - Returns: - tuple: A tuple containing: - - logp (torch.Tensor): Log probability scores for the input sequence, - shape (sequence_length, vocab_size). - - state (List[Any]): Updated decoder state. - - Note: - This method supports multi-modal scoring by incorporating both text and speech - inputs when available. The speech attention mechanism is used only if - use_speech_attn is True and speech input is provided. It is typically used - in beam search and other decoding algorithms to evaluate candidate sequences. - """ + """Score.""" ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) logp, state = self.forward_one_step( ys.unsqueeze(0), @@ -1187,30 +880,19 @@ def batch_score( xs: torch.Tensor, speech: torch.Tensor = None, ) -> Tuple[torch.Tensor, List[Any]]: - """ - Score a batch of token sequences. - - This method computes the log probability scores for a batch of token sequences, - optionally incorporating speech input for multi-modal scoring. + """Score new token batch. Args: - ys (torch.Tensor): Batch of token sequences, shape (batch_size, sequence_length). - states (List[Any]): List of scorer states for prefix tokens. - xs (torch.Tensor): Batch of encoder outputs, shape (batch_size, max_length, encoder_output_size). - speech (torch.Tensor, optional): Batch of speech inputs, shape (batch_size, speech_maxlen, speech_feat). + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). Returns: - tuple: A tuple containing: - - logp (torch.Tensor): Batch of log probability scores for the next token, - shape (batch_size, vocab_size). - - state_list (List[List[Any]]): Updated list of scorer states for each sequence in the batch. - - Note: - This method is optimized for batch processing, which is more efficient than - scoring sequences individually. It supports multi-modal scoring by incorporating - both text and speech inputs when available. The speech attention mechanism is used - only if use_speech_attn is True and speech input is provided. This method is - particularly useful for beam search or batch decoding in multi-modal tasks. + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + """ # merge states n_batch = len(ys) diff --git a/espnet2/asr/decoder/whisper_decoder.py b/espnet2/asr/decoder/whisper_decoder.py index 17c157252ee..b0106bd1bf5 100644 --- a/espnet2/asr/decoder/whisper_decoder.py +++ b/espnet2/asr/decoder/whisper_decoder.py @@ -9,34 +9,6 @@ class ExpandedTokenEmbedding(torch.nn.Module): - """ - A custom embedding layer that expands an existing embedding with additional tokens. - - This class extends the functionality of a given embedding layer by adding - more token embeddings while preserving the original embeddings. The new - embeddings are initialized with normal distribution based on the statistics - of the original embedding weights. - - Attributes: - ori_emb (torch.nn.Embedding): The original embedding layer. - add_emb (torch.nn.Embedding): The additional embedding layer for new tokens. - num_embeddings (int): Total number of embeddings (original + additional). - - Args: - ori_emebedding (torch.nn.Embedding): The original embedding layer to be expanded. - additional_size (int): Number of additional token embeddings to add. - - Note: - The forward method is overridden to use the combined weights of both - original and additional embeddings. - - Example: - >>> original_embedding = torch.nn.Embedding(1000, 300) - >>> expanded_embedding = ExpandedTokenEmbedding(original_embedding, 500) - >>> input_tensor = torch.LongTensor([0, 1500, 999]) - >>> output = expanded_embedding(input_tensor) - """ - def __init__(self, ori_emebedding, additional_size): super().__init__() self.ori_emb = ori_emebedding @@ -52,42 +24,9 @@ def __init__(self, ori_emebedding, additional_size): @property def weight(self): - """ - Combined weight tensor of the original and additional embeddings. - - Returns: - torch.Tensor: A tensor containing the concatenated weights of the original - embedding (ori_emb) and the additional embedding (add_emb) along dimension 0. - - Note: - This property is used to provide a unified view of the entire embedding - weight, including both original and additional token embeddings. - """ return torch.cat([self.ori_emb.weight, self.add_emb.weight], dim=0) def forward(self, input): - """ - Performs a forward pass through the expanded embedding layer. - - This method applies the embedding operation using the combined weights - of the original and additional embeddings. It preserves the properties - of the original embedding layer, such as padding_idx, max_norm, etc. - - Args: - input (torch.Tensor): Input tensor containing token indices. - - Returns: - torch.Tensor: The embedded representation of the input tokens. - - Note: - This method overrides the default forward pass of torch.nn.Embedding - to use the combined weights while maintaining other embedding properties. - - Example: - >>> expanded_embedding = ExpandedTokenEmbedding(original_embedding, 500) - >>> input_tensor = torch.LongTensor([0, 1500, 999]) - >>> output = expanded_embedding(input_tensor) - """ return torch.nn.functional.embedding( input, self.weight, @@ -100,40 +39,9 @@ def forward(self, input): class OpenAIWhisperDecoder(AbsDecoder, BatchScorerInterface): - """ - A decoder class based on OpenAI's Whisper model for speech-to-text tasks. - - This class implements a Transformer-based decoder that utilizes the architecture - from OpenAI's Whisper model. It can be used for various speech recognition and - transcription tasks. - - Attributes: - decoders (whisper.model.Decoder): The Whisper model's decoder. - dropout (torch.nn.Dropout): Dropout layer for regularization. - load_origin_token_embedding (bool): Flag to load original token embeddings. - - Args: - vocab_size (int): Size of the vocabulary. - encoder_output_size (int): Size of the encoder's output. - dropout_rate (float, optional): Dropout rate. Defaults to 0.0. - whisper_model (str, optional): Whisper model size. Defaults to "small". - download_dir (str, optional): Directory to download the Whisper model. - load_origin_token_embedding (bool, optional): Whether to load original - token embeddings when expanding vocabulary. Defaults to False. - - Raises: - Exception: If the Whisper package is not properly installed. - - Note: - This class inherits from AbsDecoder and BatchScorerInterface, providing - compatibility with the ESPnet2 framework. - - Example: - >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) - >>> encoder_output = torch.randn(1, 100, 512) - >>> decoder_input = torch.LongTensor([[1, 2, 3, 4, 5]]) - >>> decoder_output, _ = decoder(encoder_output, torch.tensor([100]), - ... decoder_input, torch.tensor([5])) + """Transformer-based Speech-to-Text Decoder from OpenAI's Whisper Model: + + URL: https://github.com/openai/whisper """ @typechecked @@ -206,35 +114,22 @@ def forward( ys_in_pad: torch.Tensor, ys_in_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the OpenAI Whisper decoder. - - This method processes the encoder output and the decoder input to generate - the output token scores. + """Forward decoder. Args: - hs_pad (torch.Tensor): Encoded memory, float32 (batch, maxlen_in, feat). - hlens (torch.Tensor): Lengths of encoded sequences (batch,). - ys_in_pad (torch.Tensor): Input token ids, int64 (batch, maxlen_out). - ys_in_lens (torch.Tensor): Lengths of input sequences (batch,). - + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) Returns: - tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - x (torch.Tensor): Decoded token scores before softmax - (batch, maxlen_out, token). - - ys_in_lens (torch.Tensor): Lengths of input sequences (batch,). - - Note: - This method applies positional embedding, processes the input through - the decoder blocks, and generates the final output scores. - - Example: - >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) - >>> hs_pad = torch.randn(2, 100, 512) - >>> hlens = torch.tensor([100, 80]) - >>> ys_in_pad = torch.randint(0, 10000, (2, 20)) - >>> ys_in_lens = torch.tensor([20, 15]) - >>> output, out_lens = decoder(hs_pad, hlens, ys_in_pad, ys_in_lens) + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) """ tgt, memory = ys_in_pad, hs_pad tgt = ( @@ -265,37 +160,21 @@ def forward_one_step( *, cache: List[torch.Tensor] = None, ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """ - Perform a single forward step in the decoder. - - This method processes one step of decoding, typically used in inference - or beam search scenarios. + """Forward one step. Args: - tgt (torch.Tensor): Input token ids, int64 (batch, maxlen_out). - tgt_mask (torch.Tensor): Input token mask, (batch, maxlen_out). - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (including 1.2) - memory (torch.Tensor): Encoded memory, float32 (batch, maxlen_in, feat). - cache (List[torch.Tensor], optional): Cached output list of - (batch, max_time_out-1, size). Defaults to None. - + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + cache: cached output list of (batch, max_time_out-1, size) Returns: - tuple[torch.Tensor, None]: A tuple containing: - - y (torch.Tensor): Log probabilities of next tokens (batch, vocab_size). - - None: Placeholder for cache (currently not implemented). - - Note: - - The cache implementation is currently ignored for simplicity and correctness. - - This method applies positional embedding, processes through decoder blocks, - and generates log probabilities for the next tokens. - - Example: - >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) - >>> tgt = torch.LongTensor([[1, 2, 3]]) - >>> tgt_mask = torch.ones(1, 3, dtype=torch.bool) - >>> memory = torch.randn(1, 100, 512) - >>> output, _ = decoder.forward_one_step(tgt, tgt_mask, memory) + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) + NOTE (Shih-Lun): + cache implementation is ignored for now + for simplicity & correctness """ x = ( self.decoders.token_embedding(tgt) @@ -319,32 +198,7 @@ def forward_one_step( return y, None def score(self, ys, state, x): - """ - Calculate the score for the next token. - - This method computes the log probability scores for the next token given - the current state and encoder output. - - Args: - ys (torch.Tensor): Current token sequence. - state (Any): Current decoder state (unused in this implementation). - x (torch.Tensor): Encoder output. - - Returns: - tuple[torch.Tensor, None]: A tuple containing: - - logp (torch.Tensor): Log probability scores for the next token. - - None: Updated state (currently not implemented). - - Note: - This method is typically used in beam search or other decoding algorithms - to score possible next tokens. - - Example: - >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) - >>> ys = torch.LongTensor([1, 2, 3]) - >>> x = torch.randn(100, 512) - >>> logp, _ = decoder.score(ys, None, x) - """ + """Score.""" logp, state = self.forward_one_step( ys.unsqueeze(0), torch.empty(0), x.unsqueeze(0), cache=state # dummy mask ) @@ -353,31 +207,19 @@ def score(self, ys, state, x): def batch_score( self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor ) -> Tuple[torch.Tensor, List[Any]]: - """ - Score new token batch. - - This method computes scores for the next tokens given batched inputs of - current token sequences and encoder features. + """Score new token batch. Args: - ys (torch.Tensor): Prefix tokens, torch.int64 (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens (unused in this implementation). - xs (torch.Tensor): Encoder features that generate ys, (n_batch, xlen, n_feat). + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). Returns: - tuple[torch.Tensor, None]: A tuple containing: - - logp (torch.Tensor): Batchified scores for next tokens, shape (n_batch, n_vocab). - - None: Placeholder for next state list (currently not implemented). - - Note: - This method is designed for batch processing, which can be more efficient - than scoring individual sequences separately. - - Example: - >>> decoder = OpenAIWhisperDecoder(vocab_size=10000, encoder_output_size=512) - >>> ys = torch.LongTensor([[1, 2, 3], [4, 5, 6]]) - >>> xs = torch.randn(2, 100, 512) - >>> logp, _ = decoder.batch_score(ys, None, xs) + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + """ # batch decoding, dummy mask is passed logp, states = self.forward_one_step(ys, torch.empty(0), xs, cache=None) diff --git a/espnet2/asr/discrete_asr_espnet_model.py b/espnet2/asr/discrete_asr_espnet_model.py index ec5eede49c2..03a27b75a9c 100644 --- a/espnet2/asr/discrete_asr_espnet_model.py +++ b/espnet2/asr/discrete_asr_espnet_model.py @@ -28,26 +28,7 @@ def autocast(enabled=True): class ESPnetDiscreteASRModel(ESPnetMTModel): - """ - Encoder-Decoder model for Automatic Speech Recognition (ASR) with discrete units. - - This class extends the ESPnetMTModel to incorporate ASR-specific components such as - CTC (Connectionist Temporal Classification) and SpecAugment. It combines encoder-decoder - architecture with CTC and attention-based decoding for improved ASR performance. - - Attributes: - specaug (Optional[AbsSpecAug]): SpecAugment module for data augmentation. - blank_id (int): ID for the blank token in CTC. - ctc_weight (float): Weight of CTC loss in the total loss calculation. - interctc_weight (float): Weight of intermediate CTC loss. - ctc (Optional[CTC]): CTC module for ASR. - error_calculator (Optional[ASRErrorCalculator]): Calculator for ASR errors. - - Note: - This model supports both regular and intermediate CTC losses, as well as - attention-based decoding. The final loss is a weighted combination of CTC - and attention decoder losses. - """ + """Encoder-Decoder model""" @typechecked def __init__( @@ -130,26 +111,14 @@ def forward( src_text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """ - Perform forward pass of the model: Frontend + Encoder + Decoder + Loss calculation. + """Frontend + Encoder + Decoder + Calc loss Args: - text (torch.Tensor): Target text tensor. Shape: (Batch, Length) - text_lengths (torch.Tensor): Length of each sequence in the target text. Shape: (Batch,) - src_text (torch.Tensor): Source text tensor. Shape: (Batch, length) - src_text_lengths (torch.Tensor): Length of each sequence in the source text. Shape: (Batch,) - **kwargs: Additional keyword arguments. "utt_id" is expected among the inputs. - - Returns: - Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: A tuple containing: - - loss (torch.Tensor): The total loss of the model. - - stats (Dict[str, torch.Tensor]): A dictionary with various statistics and metrics. - - weight (torch.Tensor): The batch size as a weight for the loss. - - Note: - This method handles the entire forward pass of the model, including CTC and - attention-based loss calculations. It also computes various metrics such as - Character Error Rate (CER) and Word Error Rate (WER) when not in training mode. + text: (Batch, Length) + text_lengths: (Batch,) + src_text: (Batch, length) + src_text_lengths: (Batch,) + kwargs: "utt_id" is among the input. """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -234,31 +203,11 @@ def forward( def encode( self, src_text: torch.Tensor, src_text_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Perform frontend processing and encoding of the input. - - This method applies the frontend processing, optional SpecAugment, pre-encoding, - main encoding, and post-encoding steps to the input. + """Frontend + Encoder. Note that this method is used by mt_inference.py Args: - src_text (torch.Tensor): Source text tensor. Shape: (Batch, Length, ...) - src_text_lengths (torch.Tensor): Length of each sequence in the source text. Shape: (Batch,) - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - encoder_out (torch.Tensor): The encoded output. Shape: (Batch, Length2, Dim2) - - encoder_out_lens (torch.Tensor): The length of each sequence in the encoded output. - - If intermediate outputs are available: - Tuple[Tuple[torch.Tensor, List[Tuple[int, torch.Tensor]]], torch.Tensor]: A tuple containing: - - (encoder_out, intermediate_outs): - - encoder_out (torch.Tensor): The final encoded output. - - intermediate_outs (List[Tuple[int, torch.Tensor]]): Intermediate encoder outputs. - - encoder_out_lens (torch.Tensor): The length of each sequence in the encoded output. - - Note: - This method is used by mt_inference.py and handles the entire encoding process, - including optional intermediate CTC outputs if the encoder supports it. + src_text: (Batch, Length, ...) + src_text_lengths: (Batch, ) """ with autocast(False): # 1. Extract feats diff --git a/espnet2/asr/encoder/abs_encoder.py b/espnet2/asr/encoder/abs_encoder.py index b57d195b3a3..22a1a103458 100644 --- a/espnet2/asr/encoder/abs_encoder.py +++ b/espnet2/asr/encoder/abs_encoder.py @@ -5,62 +5,8 @@ class AbsEncoder(torch.nn.Module, ABC): - """ - Abstract base class for encoders in a neural network. - - This class defines the interface for encoders used in various neural network - architectures. It inherits from both torch.nn.Module and ABC (Abstract Base Class), - ensuring that it can be used as a PyTorch module while also enforcing the - implementation of abstract methods in derived classes. - - Attributes: - None - - Note: - This class should not be instantiated directly. Instead, it should be - subclassed with concrete implementations of the abstract methods. - - Examples: - ```python - class ConcreteEncoder(AbsEncoder): - def output_size(self) -> int: - return 256 - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - # Implementation details - pass - ``` - """ - @abstractmethod def output_size(self) -> int: - """ - Returns the output size of the encoder. - - This abstract method should be implemented by subclasses to specify - the size of the encoder's output. - - Returns: - int: The size of the encoder's output. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Example: - ```python - class ConcreteEncoder(AbsEncoder): - def output_size(self) -> int: - return 256 # Example output size - - encoder = ConcreteEncoder() - size = encoder.output_size() # Returns 256 - ``` - """ raise NotImplementedError @abstractmethod @@ -70,43 +16,4 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Performs the forward pass of the encoder. - - This abstract method should be implemented by subclasses to define the - forward pass of the encoder. - - Args: - xs_pad (torch.Tensor): Padded input tensor. - ilens (torch.Tensor): Input lengths tensor. - prev_states (torch.Tensor, optional): Previous encoder states. Defaults to None. - - Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - Encoded output tensor - - Output lengths tensor - - Optional updated encoder states - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Example: - ```python - class ConcreteEncoder(AbsEncoder): - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - # Example implementation - encoded_output = self.encode(xs_pad) - output_lengths = self.calculate_lengths(ilens) - new_states = self.update_states(prev_states) - return encoded_output, output_lengths, new_states - - encoder = ConcreteEncoder() - output, lengths, states = encoder.forward(input_tensor, input_lengths) - ``` - """ raise NotImplementedError diff --git a/espnet2/asr/encoder/branchformer_encoder.py b/espnet2/asr/encoder/branchformer_encoder.py index c1d474d0df0..568a545d483 100644 --- a/espnet2/asr/encoder/branchformer_encoder.py +++ b/espnet2/asr/encoder/branchformer_encoder.py @@ -47,25 +47,19 @@ class BranchformerEncoderLayer(torch.nn.Module): - """ - Branchformer encoder layer module. - - This layer combines self-attention and convolutional gating MLP branches - to capture both global and local context. + """Branchformer encoder layer module. Args: - size (int): Model dimension. - attn (Optional[torch.nn.Module]): Self-attention module. - cgmlp (Optional[torch.nn.Module]): Convolutional Gating MLP module. - dropout_rate (float): Dropout probability. - merge_method (str): Method to merge branches ('concat', 'learned_ave', or 'fixed_ave'). - cgmlp_weight (float): Weight of the CGMLP branch for 'fixed_ave' merge method. Default: 0.5. - attn_branch_drop_rate (float): Probability of dropping the attention branch. Default: 0.0. - stochastic_depth_rate (float): Stochastic depth probability. Default: 0.0. - - Note: - At least one of `attn` or `cgmlp` must be provided. - The merge_method determines how the outputs of the two branches are combined. + size (int): model dimension + attn: standard self-attention or efficient attention, optional + cgmlp: ConvolutionalGatingMLP, optional + dropout_rate (float): dropout probability + merge_method (str): concat, learned_ave, fixed_ave + cgmlp_weight (float): weight of the cgmlp branch, between 0 and 1, + used if merge_method is fixed_ave + attn_branch_drop_rate (float): probability of dropping the attn branch, + used if merge_method is learned_ave + stochastic_depth_rate (float): stochastic depth probability """ def __init__( @@ -142,35 +136,18 @@ def __init__( self.merge_proj = torch.nn.Identity() def forward(self, x_input, mask, cache=None): - """ - Compute encoded features. + """Compute encoded features. Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o positional embedding. + x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - w/o pos emb: Tensor (#batch, time, size). mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (torch.Tensor, optional): Cache tensor of the input (#batch, time - 1, size). - Default: None. + cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). Returns: - Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor]]: - - If positional embedding is used: - Tuple containing: - - torch.Tensor: Output tensor with positional embedding (#batch, time, size). - - torch.Tensor: Mask tensor (#batch, time). - - If positional embedding is not used: - Tuple containing: - - torch.Tensor: Output tensor (#batch, time, size). - - torch.Tensor: Mask tensor (#batch, time). - - Raises: - NotImplementedError: If cache is not None (caching is not implemented). - - Note: - This method applies the Branchformer encoding, combining self-attention - and convolutional gating MLP branches based on the specified merge method. - It also handles stochastic depth if enabled during training. + torch.Tensor: Output tensor (#batch, time, size). + torch.Tensor: Mask tensor (#batch, time). """ if cache is not None: @@ -317,42 +294,7 @@ def forward(self, x_input, mask, cache=None): class BranchformerEncoder(AbsEncoder): - """ - Branchformer encoder module. - - This encoder combines self-attention and convolutional gating MLP in parallel branches - to capture both global and local context for speech recognition and understanding tasks. - - Args: - input_size (int): Dimension of input features. - output_size (int): Dimension of output features. Default: 256. - use_attn (bool): Whether to use attention branch. Default: True. - attention_heads (int): Number of attention heads. Default: 4. - attention_layer_type (str): Type of attention layer. Default: "rel_selfattn". - pos_enc_layer_type (str): Type of positional encoding. Default: "rel_pos". - rel_pos_type (str): Type of relative positional encoding. Default: "latest". - use_cgmlp (bool): Whether to use convolutional gating MLP branch. Default: True. - cgmlp_linear_units (int): Number of units in CGMLP linear layers. Default: 2048. - cgmlp_conv_kernel (int): Kernel size for CGMLP convolution. Default: 31. - use_linear_after_conv (bool): Whether to use linear layer after convolution in CGMLP. Default: False. - gate_activation (str): Activation function for CGMLP gate. Default: "identity". - merge_method (str): Method to merge branches. Default: "concat". - cgmlp_weight (Union[float, List[float]]): Weight(s) for CGMLP branch. Default: 0.5. - attn_branch_drop_rate (Union[float, List[float]]): Dropout rate(s) for attention branch. Default: 0.0. - num_blocks (int): Number of encoder blocks. Default: 12. - dropout_rate (float): Dropout rate. Default: 0.1. - positional_dropout_rate (float): Dropout rate for positional encoding. Default: 0.1. - attention_dropout_rate (float): Dropout rate for attention. Default: 0.0. - input_layer (Optional[str]): Type of input layer. Default: "conv2d". - zero_triu (bool): Whether to zero out the upper triangular part of attention matrix. Default: False. - padding_idx (int): Padding index for embedding layer. Default: -1. - stochastic_depth_rate (Union[float, List[float]]): Stochastic depth rate(s). Default: 0.0. - - Note: - This implementation is based on the paper "Branchformer: Parallel MLP-Attention - Architectures to Capture Local and Global Context for Speech Recognition and Understanding" - by Yifan Peng, et al. - """ + """Branchformer encoder module.""" @typechecked def __init__( @@ -563,16 +505,6 @@ def __init__( self.after_norm = LayerNorm(output_size) def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The dimension of the output features. - - Note: - This method returns the size of the encoder's output, - which is set during the initialization of the BranchformerEncoder. - """ return self._output_size def forward( @@ -581,26 +513,18 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Calculate forward propagation. + """Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor, optional): Not used in current implementation. Default: None. + prev_states (torch.Tensor): Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - torch.Tensor: Output tensor (#batch, L, output_size). - - torch.Tensor: Output length (#batch). - - Optional[torch.Tensor]: Always None in current implementation. - - Raises: - TooShortUttError: If the input is too short for subsampling. + torch.Tensor: Output tensor (#batch, L, output_size). + torch.Tensor: Output length (#batch). + torch.Tensor: Not to be used now. - Note: - The method applies the Branchformer encoding to the input tensor, - handling potential subsampling and masking as needed. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/conformer_encoder.py b/espnet2/asr/encoder/conformer_encoder.py index 856a57a8ec3..32f2134c561 100644 --- a/espnet2/asr/encoder/conformer_encoder.py +++ b/espnet2/asr/encoder/conformer_encoder.py @@ -50,31 +50,38 @@ class ConformerEncoder(AbsEncoder): - """ - Conformer encoder module. - - This class implements the Conformer encoder, which combines self-attention and - convolution to model both global and local dependencies of an input sequence. - It is designed for speech recognition tasks and can be used as a powerful - feature extractor in various speech processing applications. - - The Conformer architecture integrates components from Transformers and - Convolutional Neural Networks (CNNs) to capture both long-range and local - dependencies in the input signal. It includes multi-headed self-attention - mechanisms, convolution modules, and feed-forward layers, along with - various normalization and regularization techniques. - - Key features of the ConformerEncoder include: - - Flexible input layer options (linear, conv2d, embed) - - Configurable number of encoder blocks - - Support for relative or absolute positional encodings - - Macaron-style feed-forward layers - - Optional convolution module in each block - - Stochastic depth and layer drop for regularization - - Intermediate CTC (Connectionist Temporal Classification) integration - - The encoder can be customized through numerous parameters to adapt to - different input sizes, model capacities, and specific task requirements. + """Conformer encoder module. + + Args: + input_size (int): Input dimension. + output_size (int): Dimension of attention. + attention_heads (int): The number of heads of multi head attention. + linear_units (int): The number of units of position-wise feed forward. + num_blocks (int): The number of decoder blocks. + dropout_rate (float): Dropout rate. + attention_dropout_rate (float): Dropout rate in attention. + positional_dropout_rate (float): Dropout rate after adding positional encoding. + input_layer (Union[str, torch.nn.Module]): Input layer type. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + If True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + If False, no additional linear will be applied. i.e. x -> x + att(x) + positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". + positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. + rel_pos_type (str): Whether to use the latest relative positional encoding or + the legacy one. The legacy relative positional encoding will be deprecated + in the future. More Details can be found in + https://github.com/espnet/espnet/pull/2816. + encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. + encoder_attn_layer_type (str): Encoder attention layer type. + activation_type (str): Encoder activation function type. + macaron_style (bool): Whether to use macaron style for positionwise layer. + use_cnn_module (bool): Whether to use convolution module. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. + cnn_module_kernel (int): Kernerl size of convolution module. + padding_idx (int): Padding idx for input_layer=embed. + """ @typechecked @@ -294,16 +301,6 @@ def __init__( self.ctc_trim = ctc_trim def output_size(self) -> int: - """ - Returns the output size of the encoder. - - Returns: - int: The dimension of the encoder output. - - Note: - This method returns the value of the internal `_output_size` attribute, - which is typically set during the initialization of the encoder. - """ return self._output_size def forward( @@ -314,34 +311,20 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Calculate forward propagation. + """Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor, optional): Not used in current implementation. - ctc (CTC, optional): CTC module for intermediate CTC loss. - return_all_hs (bool, optional): Whether to return all hidden states. + prev_states (torch.Tensor): Not to be used now. + ctc (CTC): ctc module for intermediate CTC loss + return_all_hs (bool): whether to return all hidden states Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - torch.Tensor: Output tensor (#batch, L, output_size). - - torch.Tensor: Output length (#batch). - - Optional[torch.Tensor]: Not used in current implementation. - - Raises: - TooShortUttError: If the input is too short for subsampling. - - Note: - If intermediate CTC is used, the output tensor will be a tuple containing - the final output and a list of intermediate outputs. - - Examples: - >>> encoder = ConformerEncoder(input_size=80, output_size=256) - >>> x = torch.randn(2, 100, 80) - >>> ilens = torch.tensor([100, 80]) - >>> output, out_lens, _ = encoder(x, ilens) + torch.Tensor: Output tensor (#batch, L, output_size). + torch.Tensor: Output length (#batch). + torch.Tensor: Not to be used now. + """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/contextual_block_conformer_encoder.py b/espnet2/asr/encoder/contextual_block_conformer_encoder.py index bb4ea63ed78..f8a761ac70a 100644 --- a/espnet2/asr/encoder/contextual_block_conformer_encoder.py +++ b/espnet2/asr/encoder/contextual_block_conformer_encoder.py @@ -1,675 +1,596 @@ -# -*- coding: utf-8 -*- -""" -Created on Sat Aug 21 17:27:16 2021. - -@author: Keqi Deng (UCAS) -""" - -import math -from typing import Optional, Tuple - -import torch -from typeguard import typechecked - -from espnet2.asr.encoder.abs_encoder import AbsEncoder -from espnet.nets.pytorch_backend.conformer.contextual_block_encoder_layer import ( - ContextualBlockEncoderLayer, -) -from espnet.nets.pytorch_backend.conformer.convolution import ConvolutionModule -from espnet.nets.pytorch_backend.nets_utils import get_activation, make_pad_mask -from espnet.nets.pytorch_backend.transformer.attention import MultiHeadedAttention -from espnet.nets.pytorch_backend.transformer.embedding import StreamPositionalEncoding -from espnet.nets.pytorch_backend.transformer.layer_norm import LayerNorm -from espnet.nets.pytorch_backend.transformer.multi_layer_conv import ( - Conv1dLinear, - MultiLayeredConv1d, -) -from espnet.nets.pytorch_backend.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from espnet.nets.pytorch_backend.transformer.repeat import repeat -from espnet.nets.pytorch_backend.transformer.subsampling_without_posenc import ( - Conv2dSubsamplingWOPosEnc, -) - - -class ContextualBlockConformerEncoder(AbsEncoder): - """ - Contextual Block Conformer encoder module. - - This class implements a Conformer encoder with contextual block processing, - which allows for efficient processing of long sequences by dividing them - into smaller blocks and incorporating context information. - - Args: - input_size (int): Dimension of input features. - output_size (int, optional): Dimension of attention and output features. Defaults to 256. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in position-wise feed forward layers. Defaults to 2048. - num_blocks (int, optional): Number of encoder blocks. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. - input_layer (str, optional): Type of input layer. Defaults to "conv2d". - normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to True. - concat_after (bool, optional): Whether to concat attention layer's input and output. Defaults to False. - positionwise_layer_type (str, optional): Type of position-wise layer. Defaults to "linear". - positionwise_conv_kernel_size (int, optional): Kernel size of position-wise conv1d layer. Defaults to 3. - macaron_style (bool, optional): Whether to use macaron style for positionwise layer. Defaults to False. - pos_enc_class (class, optional): Positional encoding class. Defaults to StreamPositionalEncoding. - selfattention_layer_type (str, optional): Type of self-attention layer. Defaults to "rel_selfattn". - activation_type (str, optional): Type of activation function. Defaults to "swish". - use_cnn_module (bool, optional): Whether to use CNN module. Defaults to True. - cnn_module_kernel (int, optional): Kernel size of CNN module. Defaults to 31. - padding_idx (int, optional): Padding index for input layer. Defaults to -1. - block_size (int, optional): Block size for contextual block processing. Defaults to 40. - hop_size (int, optional): Hop size for block processing. Defaults to 16. - look_ahead (int, optional): Look-ahead size for block processing. Defaults to 16. - init_average (bool, optional): Whether to use average as initial context. Defaults to True. - ctx_pos_enc (bool, optional): Whether to use positional encoding for context vectors. Defaults to True. - - Attributes: - block_size (int): Block size for contextual block processing. - hop_size (int): Hop size for block processing. - look_ahead (int): Look-ahead size for block processing. - init_average (bool): Whether to use average as initial context. - ctx_pos_enc (bool): Whether to use positional encoding for context vectors. - """ - - @typechecked - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 3, - macaron_style: bool = False, - pos_enc_class=StreamPositionalEncoding, - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - cnn_module_kernel: int = 31, - padding_idx: int = -1, - block_size: int = 40, - hop_size: int = 16, - look_ahead: int = 16, - init_average: bool = True, - ctx_pos_enc: bool = True, - ): - super().__init__() - self._output_size = output_size - self.pos_enc = pos_enc_class(output_size, positional_dropout_rate) - activation = get_activation(activation_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - ) - self.subsample = 1 - elif input_layer == "conv2d": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, output_size, dropout_rate, kernels=[3, 3], strides=[2, 2] - ) - self.subsample = 4 - elif input_layer == "conv2d6": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, output_size, dropout_rate, kernels=[3, 5], strides=[2, 3] - ) - self.subsample = 6 - elif input_layer == "conv2d8": - self.embed = Conv2dSubsamplingWOPosEnc( - input_size, - output_size, - dropout_rate, - kernels=[3, 3, 3], - strides=[2, 2, 2], - ) - self.subsample = 8 - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - ) - self.subsample = 1 - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate), - ) - self.subsample = 1 - elif input_layer is None: - self.embed = torch.nn.Sequential( - pos_enc_class(output_size, positional_dropout_rate) - ) - self.subsample = 1 - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - convolution_layer = ConvolutionModule - convolution_layer_args = (output_size, cnn_module_kernel, activation) - - self.encoders = repeat( - num_blocks, - lambda lnum: ContextualBlockEncoderLayer( - output_size, - MultiHeadedAttention( - attention_heads, output_size, attention_dropout_rate - ), - positionwise_layer(*positionwise_layer_args), - positionwise_layer(*positionwise_layer_args) if macaron_style else None, - convolution_layer(*convolution_layer_args) if use_cnn_module else None, - dropout_rate, - num_blocks, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - # for block processing - self.block_size = block_size - self.hop_size = hop_size - self.look_ahead = look_ahead - self.init_average = init_average - self.ctx_pos_enc = ctx_pos_enc - - def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The output size (dimension) of the encoder. - """ - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - is_final=True, - infer_mode=False, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the Contextual Block Conformer Encoder. - - This method processes the input tensor and returns the encoded output. It handles both - training/validation and inference modes. - - Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, - L is sequence length, and D is input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - prev_states (torch.Tensor, optional): Previous encoder states for streaming inference. - Defaults to None. - is_final (bool, optional): Whether this is the final block in streaming inference. - Defaults to True. - infer_mode (bool, optional): Whether to use inference mode. Defaults to False. - - Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - Encoded output tensor of shape (B, L, D_out), where D_out is the output dimension. - - Output lengths of shape (B,). - - Updated encoder states (None if not in inference mode). - - Note: - The behavior of this method differs between training/validation and inference modes. - In training/validation, it processes the entire input sequence at once. - In inference mode, it supports streaming processing of input chunks. - """ - if self.training or not infer_mode: - return self.forward_train(xs_pad, ilens, prev_states) - else: - return self.forward_infer(xs_pad, ilens, prev_states, is_final) - - def forward_train( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass for training and validation. - - This method processes the entire input sequence at once, applying the contextual - block processing technique for long sequences. - - Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, - L is sequence length, and D is input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - prev_states (torch.Tensor, optional): Not used in training mode. Defaults to None. - - Returns: - Tuple[torch.Tensor, torch.Tensor, None]: A tuple containing: - - Encoded output tensor of shape (B, L, D_out), where D_out is the output dimension. - - Output lengths of shape (B,). - - None (placeholder for consistency with inference mode). - - Note: - This method implements the core contextual block processing algorithm, including: - - Subsampling and positional encoding of input - - Creation and processing of context vectors - - Block-wise processing of the input sequence - - Aggregation of block outputs into the final encoded sequence - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): - xs_pad, masks = self.embed(xs_pad, masks) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - # create empty output container - total_frame_num = xs_pad.size(1) - ys_pad = xs_pad.new_zeros(xs_pad.size()) - - past_size = self.block_size - self.hop_size - self.look_ahead - - # block_size could be 0 meaning infinite - # apply usual encoder for short sequence - if self.block_size == 0 or total_frame_num <= self.block_size: - xs_pad, masks, _, _, _, _, _ = self.encoders( - self.pos_enc(xs_pad), masks, False, None, None - ) - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - return xs_pad, olens, None - - # start block processing - cur_hop = 0 - block_num = math.ceil( - float(total_frame_num - past_size - self.look_ahead) / float(self.hop_size) - ) - bsize = xs_pad.size(0) - addin = xs_pad.new_zeros( - bsize, block_num, xs_pad.size(-1) - ) # additional context embedding vecctors - - # first step - if self.init_average: # initialize with average value - addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).mean(1) - else: # initialize with max value - addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).max(1) - cur_hop += self.hop_size - # following steps - while cur_hop + self.block_size < total_frame_num: - if self.init_average: # initialize with average value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, self.block_size - ).mean(1) - else: # initialize with max value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, self.block_size - ).max(1) - cur_hop += self.hop_size - # last step - if cur_hop < total_frame_num and cur_hop // self.hop_size < block_num: - if self.init_average: # initialize with average value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, total_frame_num - cur_hop - ).mean(1) - else: # initialize with max value - addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( - 1, cur_hop, total_frame_num - cur_hop - ).max(1) - - if self.ctx_pos_enc: - addin = self.pos_enc(addin) - - xs_pad = self.pos_enc(xs_pad) - - # set up masks - mask_online = xs_pad.new_zeros( - xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 - ) - mask_online.narrow(2, 1, self.block_size + 1).narrow( - 3, 0, self.block_size + 1 - ).fill_(1) - - xs_chunk = xs_pad.new_zeros( - bsize, block_num, self.block_size + 2, xs_pad.size(-1) - ) - - # fill the input - # first step - left_idx = 0 - block_idx = 0 - xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( - -2, left_idx, self.block_size - ) - left_idx += self.hop_size - block_idx += 1 - # following steps - while left_idx + self.block_size < total_frame_num and block_idx < block_num: - xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( - -2, left_idx, self.block_size - ) - left_idx += self.hop_size - block_idx += 1 - # last steps - last_size = total_frame_num - left_idx - xs_chunk[:, block_idx, 1 : last_size + 1] = xs_pad.narrow( - -2, left_idx, last_size - ) - - # fill the initial context vector - xs_chunk[:, 0, 0] = addin[:, 0] - xs_chunk[:, 1:, 0] = addin[:, 0 : block_num - 1] - xs_chunk[:, :, self.block_size + 1] = addin - - # forward - ys_chunk, mask_online, _, _, _, _, _ = self.encoders( - xs_chunk, mask_online, False, xs_chunk - ) - - # copy output - # first step - offset = self.block_size - self.look_ahead - self.hop_size + 1 - left_idx = 0 - block_idx = 0 - cur_hop = self.block_size - self.look_ahead - ys_pad[:, left_idx:cur_hop] = ys_chunk[:, block_idx, 1 : cur_hop + 1] - left_idx += self.hop_size - block_idx += 1 - # following steps - while left_idx + self.block_size < total_frame_num and block_idx < block_num: - ys_pad[:, cur_hop : cur_hop + self.hop_size] = ys_chunk[ - :, block_idx, offset : offset + self.hop_size - ] - cur_hop += self.hop_size - left_idx += self.hop_size - block_idx += 1 - ys_pad[:, cur_hop:total_frame_num] = ys_chunk[ - :, block_idx, offset : last_size + 1, : - ] - - if self.normalize_before: - ys_pad = self.after_norm(ys_pad) - - olens = masks.squeeze(1).sum(1) - return ys_pad, olens, None - - def forward_infer( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - is_final: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass for inference (decoding) with streaming support. - - This method processes input chunks sequentially, maintaining necessary states - for continuous streaming inference. - - Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is batch size, - L is sequence length, and D is input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - prev_states (torch.Tensor, optional): Previous encoder states from the last call. - Contains buffers and processing information. Defaults to None. - is_final (bool, optional): Whether this is the final chunk of the utterance. - Defaults to True. - - Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[Dict]]: A tuple containing: - - Encoded output tensor of shape (B, L', D_out), where L' is the processed - length and D_out is the output dimension. - - Output lengths of shape (B,). - - Updated encoder states (Dict) for the next call, or None if is_final is True. - - Note: - This method implements streaming inference by: - - Managing buffers for input and output - - Applying block-wise processing on the input chunk - - Handling overlap between consecutive chunks - - Maintaining context information across calls - - Supporting final processing when the last chunk is received - - The returned states dict includes: - - prev_addin: Previous additional input for context - - buffer_before_downsampling: Input buffer before downsampling - - ilens_buffer: Input length buffer - - buffer_after_downsampling: Input buffer after downsampling - - n_processed_blocks: Number of processed blocks - - past_encoder_ctx: Past encoder context - """ - if prev_states is None: - prev_addin = None - buffer_before_downsampling = None - ilens_buffer = None - buffer_after_downsampling = None - n_processed_blocks = 0 - past_encoder_ctx = None - else: - prev_addin = prev_states["prev_addin"] - buffer_before_downsampling = prev_states["buffer_before_downsampling"] - ilens_buffer = prev_states["ilens_buffer"] - buffer_after_downsampling = prev_states["buffer_after_downsampling"] - n_processed_blocks = prev_states["n_processed_blocks"] - past_encoder_ctx = prev_states["past_encoder_ctx"] - bsize = xs_pad.size(0) - assert bsize == 1 - - if prev_states is not None: - xs_pad = torch.cat([buffer_before_downsampling, xs_pad], dim=1) - ilens += ilens_buffer - - if is_final: - buffer_before_downsampling = None - else: - n_samples = xs_pad.size(1) // self.subsample - 1 - if n_samples < 2: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": xs_pad, - "ilens_buffer": ilens, - "buffer_after_downsampling": buffer_after_downsampling, - "n_processed_blocks": n_processed_blocks, - "past_encoder_ctx": past_encoder_ctx, - } - return ( - xs_pad.new_zeros(bsize, 0, self._output_size), - xs_pad.new_zeros(bsize), - next_states, - ) - - n_res_samples = xs_pad.size(1) % self.subsample + self.subsample * 2 - buffer_before_downsampling = xs_pad.narrow( - 1, xs_pad.size(1) - n_res_samples, n_res_samples - ) - xs_pad = xs_pad.narrow(1, 0, n_samples * self.subsample) - - ilens_buffer = ilens.new_full( - [1], dtype=torch.long, fill_value=n_res_samples - ) - ilens = ilens.new_full( - [1], dtype=torch.long, fill_value=n_samples * self.subsample - ) - - if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): - xs_pad, _ = self.embed(xs_pad, None) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - # create empty output container - if buffer_after_downsampling is not None: - xs_pad = torch.cat([buffer_after_downsampling, xs_pad], dim=1) - - total_frame_num = xs_pad.size(1) - - if is_final: - past_size = self.block_size - self.hop_size - self.look_ahead - block_num = math.ceil( - float(total_frame_num - past_size - self.look_ahead) - / float(self.hop_size) - ) - buffer_after_downsampling = None - else: - if total_frame_num <= self.block_size: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": buffer_before_downsampling, - "ilens_buffer": ilens_buffer, - "buffer_after_downsampling": xs_pad, - "n_processed_blocks": n_processed_blocks, - "past_encoder_ctx": past_encoder_ctx, - } - return ( - xs_pad.new_zeros(bsize, 0, self._output_size), - xs_pad.new_zeros(bsize), - next_states, - ) - - overlap_size = self.block_size - self.hop_size - block_num = max(0, xs_pad.size(1) - overlap_size) // self.hop_size - res_frame_num = xs_pad.size(1) - self.hop_size * block_num - buffer_after_downsampling = xs_pad.narrow( - 1, xs_pad.size(1) - res_frame_num, res_frame_num - ) - xs_pad = xs_pad.narrow(1, 0, block_num * self.hop_size + overlap_size) - - # block_size could be 0 meaning infinite - # apply usual encoder for short sequence - assert self.block_size > 0 - if n_processed_blocks == 0 and total_frame_num <= self.block_size and is_final: - xs_chunk = self.pos_enc(xs_pad).unsqueeze(1) - xs_pad, _, _, _, _, _, _ = self.encoders( - xs_chunk, None, True, None, None, True - ) - xs_pad = xs_pad.squeeze(0) - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - return xs_pad, xs_pad.new_zeros(bsize), None - # return xs_pad, None, None - - # start block processing - xs_chunk = xs_pad.new_zeros( - bsize, block_num, self.block_size + 2, xs_pad.size(-1) - ) - - for i in range(block_num): - cur_hop = i * self.hop_size - chunk_length = min(self.block_size, total_frame_num - cur_hop) - addin = xs_pad.narrow(1, cur_hop, chunk_length) - if self.init_average: - addin = addin.mean(1, keepdim=True) - else: - addin = addin.max(1, keepdim=True) - if self.ctx_pos_enc: - addin = self.pos_enc(addin, i + n_processed_blocks) - - if prev_addin is None: - prev_addin = addin - xs_chunk[:, i, 0] = prev_addin - xs_chunk[:, i, -1] = addin - - chunk = self.pos_enc( - xs_pad.narrow(1, cur_hop, chunk_length), - cur_hop + self.hop_size * n_processed_blocks, - ) - - xs_chunk[:, i, 1 : chunk_length + 1] = chunk - - prev_addin = addin - - # mask setup, it should be the same to that of forward_train - mask_online = xs_pad.new_zeros( - xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 - ) - mask_online.narrow(2, 1, self.block_size + 1).narrow( - 3, 0, self.block_size + 1 - ).fill_(1) - - ys_chunk, _, _, _, past_encoder_ctx, _, _ = self.encoders( - xs_chunk, mask_online, True, past_encoder_ctx - ) - - # remove addin - ys_chunk = ys_chunk.narrow(2, 1, self.block_size) - - offset = self.block_size - self.look_ahead - self.hop_size - if is_final: - if n_processed_blocks == 0: - y_length = xs_pad.size(1) - else: - y_length = xs_pad.size(1) - offset - else: - y_length = block_num * self.hop_size - if n_processed_blocks == 0: - y_length += offset - ys_pad = xs_pad.new_zeros((xs_pad.size(0), y_length, xs_pad.size(2))) - if n_processed_blocks == 0: - ys_pad[:, 0:offset] = ys_chunk[:, 0, 0:offset] - for i in range(block_num): - cur_hop = i * self.hop_size - if n_processed_blocks == 0: - cur_hop += offset - if i == block_num - 1 and is_final: - chunk_length = min(self.block_size - offset, ys_pad.size(1) - cur_hop) - else: - chunk_length = self.hop_size - ys_pad[:, cur_hop : cur_hop + chunk_length] = ys_chunk[ - :, i, offset : offset + chunk_length - ] - if self.normalize_before: - ys_pad = self.after_norm(ys_pad) - - if is_final: - next_states = None - else: - next_states = { - "prev_addin": prev_addin, - "buffer_before_downsampling": buffer_before_downsampling, - "ilens_buffer": ilens_buffer, - "buffer_after_downsampling": buffer_after_downsampling, - "n_processed_blocks": n_processed_blocks + block_num, - "past_encoder_ctx": past_encoder_ctx, - } - - return ( - ys_pad, - torch.tensor([y_length], dtype=xs_pad.dtype, device=ys_pad.device), - next_states, - ) - # return ys_pad, None, next_states +# -*- coding: utf-8 -*- +""" +Created on Sat Aug 21 17:27:16 2021. + +@author: Keqi Deng (UCAS) +""" + +import math +from typing import Optional, Tuple + +import torch +from typeguard import typechecked + +from espnet2.asr.encoder.abs_encoder import AbsEncoder +from espnet.nets.pytorch_backend.conformer.contextual_block_encoder_layer import ( + ContextualBlockEncoderLayer, +) +from espnet.nets.pytorch_backend.conformer.convolution import ConvolutionModule +from espnet.nets.pytorch_backend.nets_utils import get_activation, make_pad_mask +from espnet.nets.pytorch_backend.transformer.attention import MultiHeadedAttention +from espnet.nets.pytorch_backend.transformer.embedding import StreamPositionalEncoding +from espnet.nets.pytorch_backend.transformer.layer_norm import LayerNorm +from espnet.nets.pytorch_backend.transformer.multi_layer_conv import ( + Conv1dLinear, + MultiLayeredConv1d, +) +from espnet.nets.pytorch_backend.transformer.positionwise_feed_forward import ( + PositionwiseFeedForward, +) +from espnet.nets.pytorch_backend.transformer.repeat import repeat +from espnet.nets.pytorch_backend.transformer.subsampling_without_posenc import ( + Conv2dSubsamplingWOPosEnc, +) + + +class ContextualBlockConformerEncoder(AbsEncoder): + """Contextual Block Conformer encoder module. + + Args: + input_size: input dim + output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of decoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + positional_dropout_rate: dropout rate after adding positional encoding + input_layer: input layer type + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + positionwise_layer_type: linear of conv1d + positionwise_conv_kernel_size: kernel size of positionwise conv1d layer + padding_idx: padding_idx for input_layer=embed + block_size: block size for contextual block processing + hop_Size: hop size for block processing + look_ahead: look-ahead size for block_processing + init_average: whether to use average as initial context (otherwise max values) + ctx_pos_enc: whether to use positional encoding to the context vectors + """ + + @typechecked + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 3, + macaron_style: bool = False, + pos_enc_class=StreamPositionalEncoding, + selfattention_layer_type: str = "rel_selfattn", + activation_type: str = "swish", + use_cnn_module: bool = True, + cnn_module_kernel: int = 31, + padding_idx: int = -1, + block_size: int = 40, + hop_size: int = 16, + look_ahead: int = 16, + init_average: bool = True, + ctx_pos_enc: bool = True, + ): + super().__init__() + self._output_size = output_size + self.pos_enc = pos_enc_class(output_size, positional_dropout_rate) + activation = get_activation(activation_type) + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + ) + self.subsample = 1 + elif input_layer == "conv2d": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, output_size, dropout_rate, kernels=[3, 3], strides=[2, 2] + ) + self.subsample = 4 + elif input_layer == "conv2d6": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, output_size, dropout_rate, kernels=[3, 5], strides=[2, 3] + ) + self.subsample = 6 + elif input_layer == "conv2d8": + self.embed = Conv2dSubsamplingWOPosEnc( + input_size, + output_size, + dropout_rate, + kernels=[3, 3, 3], + strides=[2, 2, 2], + ) + self.subsample = 8 + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + ) + self.subsample = 1 + elif isinstance(input_layer, torch.nn.Module): + self.embed = torch.nn.Sequential( + input_layer, + pos_enc_class(output_size, positional_dropout_rate), + ) + self.subsample = 1 + elif input_layer is None: + self.embed = torch.nn.Sequential( + pos_enc_class(output_size, positional_dropout_rate) + ) + self.subsample = 1 + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + convolution_layer = ConvolutionModule + convolution_layer_args = (output_size, cnn_module_kernel, activation) + + self.encoders = repeat( + num_blocks, + lambda lnum: ContextualBlockEncoderLayer( + output_size, + MultiHeadedAttention( + attention_heads, output_size, attention_dropout_rate + ), + positionwise_layer(*positionwise_layer_args), + positionwise_layer(*positionwise_layer_args) if macaron_style else None, + convolution_layer(*convolution_layer_args) if use_cnn_module else None, + dropout_rate, + num_blocks, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + # for block processing + self.block_size = block_size + self.hop_size = hop_size + self.look_ahead = look_ahead + self.init_average = init_average + self.ctx_pos_enc = ctx_pos_enc + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + is_final=True, + infer_mode=False, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + infer_mode: whether to be used for inference. This is used to + distinguish between forward_train (train and validate) and + forward_infer (decode). + Returns: + position embedded tensor and mask + """ + if self.training or not infer_mode: + return self.forward_train(xs_pad, ilens, prev_states) + else: + return self.forward_infer(xs_pad, ilens, prev_states, is_final) + + def forward_train( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + + if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): + xs_pad, masks = self.embed(xs_pad, masks) + elif self.embed is not None: + xs_pad = self.embed(xs_pad) + + # create empty output container + total_frame_num = xs_pad.size(1) + ys_pad = xs_pad.new_zeros(xs_pad.size()) + + past_size = self.block_size - self.hop_size - self.look_ahead + + # block_size could be 0 meaning infinite + # apply usual encoder for short sequence + if self.block_size == 0 or total_frame_num <= self.block_size: + xs_pad, masks, _, _, _, _, _ = self.encoders( + self.pos_enc(xs_pad), masks, False, None, None + ) + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + return xs_pad, olens, None + + # start block processing + cur_hop = 0 + block_num = math.ceil( + float(total_frame_num - past_size - self.look_ahead) / float(self.hop_size) + ) + bsize = xs_pad.size(0) + addin = xs_pad.new_zeros( + bsize, block_num, xs_pad.size(-1) + ) # additional context embedding vecctors + + # first step + if self.init_average: # initialize with average value + addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).mean(1) + else: # initialize with max value + addin[:, 0, :] = xs_pad.narrow(1, cur_hop, self.block_size).max(1) + cur_hop += self.hop_size + # following steps + while cur_hop + self.block_size < total_frame_num: + if self.init_average: # initialize with average value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, self.block_size + ).mean(1) + else: # initialize with max value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, self.block_size + ).max(1) + cur_hop += self.hop_size + # last step + if cur_hop < total_frame_num and cur_hop // self.hop_size < block_num: + if self.init_average: # initialize with average value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, total_frame_num - cur_hop + ).mean(1) + else: # initialize with max value + addin[:, cur_hop // self.hop_size, :] = xs_pad.narrow( + 1, cur_hop, total_frame_num - cur_hop + ).max(1) + + if self.ctx_pos_enc: + addin = self.pos_enc(addin) + + xs_pad = self.pos_enc(xs_pad) + + # set up masks + mask_online = xs_pad.new_zeros( + xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 + ) + mask_online.narrow(2, 1, self.block_size + 1).narrow( + 3, 0, self.block_size + 1 + ).fill_(1) + + xs_chunk = xs_pad.new_zeros( + bsize, block_num, self.block_size + 2, xs_pad.size(-1) + ) + + # fill the input + # first step + left_idx = 0 + block_idx = 0 + xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( + -2, left_idx, self.block_size + ) + left_idx += self.hop_size + block_idx += 1 + # following steps + while left_idx + self.block_size < total_frame_num and block_idx < block_num: + xs_chunk[:, block_idx, 1 : self.block_size + 1] = xs_pad.narrow( + -2, left_idx, self.block_size + ) + left_idx += self.hop_size + block_idx += 1 + # last steps + last_size = total_frame_num - left_idx + xs_chunk[:, block_idx, 1 : last_size + 1] = xs_pad.narrow( + -2, left_idx, last_size + ) + + # fill the initial context vector + xs_chunk[:, 0, 0] = addin[:, 0] + xs_chunk[:, 1:, 0] = addin[:, 0 : block_num - 1] + xs_chunk[:, :, self.block_size + 1] = addin + + # forward + ys_chunk, mask_online, _, _, _, _, _ = self.encoders( + xs_chunk, mask_online, False, xs_chunk + ) + + # copy output + # first step + offset = self.block_size - self.look_ahead - self.hop_size + 1 + left_idx = 0 + block_idx = 0 + cur_hop = self.block_size - self.look_ahead + ys_pad[:, left_idx:cur_hop] = ys_chunk[:, block_idx, 1 : cur_hop + 1] + left_idx += self.hop_size + block_idx += 1 + # following steps + while left_idx + self.block_size < total_frame_num and block_idx < block_num: + ys_pad[:, cur_hop : cur_hop + self.hop_size] = ys_chunk[ + :, block_idx, offset : offset + self.hop_size + ] + cur_hop += self.hop_size + left_idx += self.hop_size + block_idx += 1 + ys_pad[:, cur_hop:total_frame_num] = ys_chunk[ + :, block_idx, offset : last_size + 1, : + ] + + if self.normalize_before: + ys_pad = self.after_norm(ys_pad) + + olens = masks.squeeze(1).sum(1) + return ys_pad, olens, None + + def forward_infer( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + is_final: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + if prev_states is None: + prev_addin = None + buffer_before_downsampling = None + ilens_buffer = None + buffer_after_downsampling = None + n_processed_blocks = 0 + past_encoder_ctx = None + else: + prev_addin = prev_states["prev_addin"] + buffer_before_downsampling = prev_states["buffer_before_downsampling"] + ilens_buffer = prev_states["ilens_buffer"] + buffer_after_downsampling = prev_states["buffer_after_downsampling"] + n_processed_blocks = prev_states["n_processed_blocks"] + past_encoder_ctx = prev_states["past_encoder_ctx"] + bsize = xs_pad.size(0) + assert bsize == 1 + + if prev_states is not None: + xs_pad = torch.cat([buffer_before_downsampling, xs_pad], dim=1) + ilens += ilens_buffer + + if is_final: + buffer_before_downsampling = None + else: + n_samples = xs_pad.size(1) // self.subsample - 1 + if n_samples < 2: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": xs_pad, + "ilens_buffer": ilens, + "buffer_after_downsampling": buffer_after_downsampling, + "n_processed_blocks": n_processed_blocks, + "past_encoder_ctx": past_encoder_ctx, + } + return ( + xs_pad.new_zeros(bsize, 0, self._output_size), + xs_pad.new_zeros(bsize), + next_states, + ) + + n_res_samples = xs_pad.size(1) % self.subsample + self.subsample * 2 + buffer_before_downsampling = xs_pad.narrow( + 1, xs_pad.size(1) - n_res_samples, n_res_samples + ) + xs_pad = xs_pad.narrow(1, 0, n_samples * self.subsample) + + ilens_buffer = ilens.new_full( + [1], dtype=torch.long, fill_value=n_res_samples + ) + ilens = ilens.new_full( + [1], dtype=torch.long, fill_value=n_samples * self.subsample + ) + + if isinstance(self.embed, Conv2dSubsamplingWOPosEnc): + xs_pad, _ = self.embed(xs_pad, None) + elif self.embed is not None: + xs_pad = self.embed(xs_pad) + + # create empty output container + if buffer_after_downsampling is not None: + xs_pad = torch.cat([buffer_after_downsampling, xs_pad], dim=1) + + total_frame_num = xs_pad.size(1) + + if is_final: + past_size = self.block_size - self.hop_size - self.look_ahead + block_num = math.ceil( + float(total_frame_num - past_size - self.look_ahead) + / float(self.hop_size) + ) + buffer_after_downsampling = None + else: + if total_frame_num <= self.block_size: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": buffer_before_downsampling, + "ilens_buffer": ilens_buffer, + "buffer_after_downsampling": xs_pad, + "n_processed_blocks": n_processed_blocks, + "past_encoder_ctx": past_encoder_ctx, + } + return ( + xs_pad.new_zeros(bsize, 0, self._output_size), + xs_pad.new_zeros(bsize), + next_states, + ) + + overlap_size = self.block_size - self.hop_size + block_num = max(0, xs_pad.size(1) - overlap_size) // self.hop_size + res_frame_num = xs_pad.size(1) - self.hop_size * block_num + buffer_after_downsampling = xs_pad.narrow( + 1, xs_pad.size(1) - res_frame_num, res_frame_num + ) + xs_pad = xs_pad.narrow(1, 0, block_num * self.hop_size + overlap_size) + + # block_size could be 0 meaning infinite + # apply usual encoder for short sequence + assert self.block_size > 0 + if n_processed_blocks == 0 and total_frame_num <= self.block_size and is_final: + xs_chunk = self.pos_enc(xs_pad).unsqueeze(1) + xs_pad, _, _, _, _, _, _ = self.encoders( + xs_chunk, None, True, None, None, True + ) + xs_pad = xs_pad.squeeze(0) + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + return xs_pad, xs_pad.new_zeros(bsize), None + # return xs_pad, None, None + + # start block processing + xs_chunk = xs_pad.new_zeros( + bsize, block_num, self.block_size + 2, xs_pad.size(-1) + ) + + for i in range(block_num): + cur_hop = i * self.hop_size + chunk_length = min(self.block_size, total_frame_num - cur_hop) + addin = xs_pad.narrow(1, cur_hop, chunk_length) + if self.init_average: + addin = addin.mean(1, keepdim=True) + else: + addin = addin.max(1, keepdim=True) + if self.ctx_pos_enc: + addin = self.pos_enc(addin, i + n_processed_blocks) + + if prev_addin is None: + prev_addin = addin + xs_chunk[:, i, 0] = prev_addin + xs_chunk[:, i, -1] = addin + + chunk = self.pos_enc( + xs_pad.narrow(1, cur_hop, chunk_length), + cur_hop + self.hop_size * n_processed_blocks, + ) + + xs_chunk[:, i, 1 : chunk_length + 1] = chunk + + prev_addin = addin + + # mask setup, it should be the same to that of forward_train + mask_online = xs_pad.new_zeros( + xs_pad.size(0), block_num, self.block_size + 2, self.block_size + 2 + ) + mask_online.narrow(2, 1, self.block_size + 1).narrow( + 3, 0, self.block_size + 1 + ).fill_(1) + + ys_chunk, _, _, _, past_encoder_ctx, _, _ = self.encoders( + xs_chunk, mask_online, True, past_encoder_ctx + ) + + # remove addin + ys_chunk = ys_chunk.narrow(2, 1, self.block_size) + + offset = self.block_size - self.look_ahead - self.hop_size + if is_final: + if n_processed_blocks == 0: + y_length = xs_pad.size(1) + else: + y_length = xs_pad.size(1) - offset + else: + y_length = block_num * self.hop_size + if n_processed_blocks == 0: + y_length += offset + ys_pad = xs_pad.new_zeros((xs_pad.size(0), y_length, xs_pad.size(2))) + if n_processed_blocks == 0: + ys_pad[:, 0:offset] = ys_chunk[:, 0, 0:offset] + for i in range(block_num): + cur_hop = i * self.hop_size + if n_processed_blocks == 0: + cur_hop += offset + if i == block_num - 1 and is_final: + chunk_length = min(self.block_size - offset, ys_pad.size(1) - cur_hop) + else: + chunk_length = self.hop_size + ys_pad[:, cur_hop : cur_hop + chunk_length] = ys_chunk[ + :, i, offset : offset + chunk_length + ] + if self.normalize_before: + ys_pad = self.after_norm(ys_pad) + + if is_final: + next_states = None + else: + next_states = { + "prev_addin": prev_addin, + "buffer_before_downsampling": buffer_before_downsampling, + "ilens_buffer": ilens_buffer, + "buffer_after_downsampling": buffer_after_downsampling, + "n_processed_blocks": n_processed_blocks + block_num, + "past_encoder_ctx": past_encoder_ctx, + } + + return ( + ys_pad, + torch.tensor([y_length], dtype=xs_pad.dtype, device=ys_pad.device), + next_states, + ) + # return ys_pad, None, next_states diff --git a/espnet2/asr/encoder/contextual_block_transformer_encoder.py b/espnet2/asr/encoder/contextual_block_transformer_encoder.py index 3437a18ffcf..a0732c8cd1f 100644 --- a/espnet2/asr/encoder/contextual_block_transformer_encoder.py +++ b/espnet2/asr/encoder/contextual_block_transformer_encoder.py @@ -30,17 +30,36 @@ class ContextualBlockTransformerEncoder(AbsEncoder): - """ - Contextual Block Transformer encoder module. - - This class implements the Contextual Block Transformer encoder as described in - Tsunoo et al. "Transformer ASR with contextual block processing" - (https://arxiv.org/abs/1910.07204). It processes input in blocks for efficient - streaming ASR and long-form audio processing. - - The encoder uses a combination of self-attention mechanisms and feed-forward - networks, with the addition of contextual block processing to handle long - sequences efficiently. + """Contextual Block Transformer encoder module. + + Details in Tsunoo et al. "Transformer ASR with contextual block processing" + (https://arxiv.org/abs/1910.07204) + + Args: + input_size: input dim + output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of encoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + positional_dropout_rate: dropout rate after adding positional encoding + input_layer: input layer type + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + positionwise_layer_type: linear of conv1d + positionwise_conv_kernel_size: kernel size of positionwise conv1d layer + padding_idx: padding_idx for input_layer=embed + block_size: block size for contextual block processing + hop_Size: hop size for block processing + look_ahead: look-ahead size for block_processing + init_average: whether to use average as initial context (otherwise max values) + ctx_pos_enc: whether to use positional encoding to the context vectors """ @typechecked @@ -160,12 +179,6 @@ def __init__( self.ctx_pos_enc = ctx_pos_enc def output_size(self) -> int: - """ - Return the output size of the encoder. - - Returns: - int: The dimension of the output features. - """ return self._output_size def forward( @@ -176,24 +189,17 @@ def forward( is_final=True, infer_mode=False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Process the input tensor and return the encoded output. - - This method handles both training and inference modes. For inference, - it uses block processing to handle streaming input efficiently. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Input tensor (B, L, D) - ilens (torch.Tensor): Input length (B) - prev_states (torch.Tensor, optional): Previous states for streaming inference. Defaults to None. - is_final (bool, optional): Whether this is the final block in streaming inference. Defaults to True. - infer_mode (bool, optional): Whether to use inference mode. Defaults to False. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + infer_mode: whether to be used for inference. This is used to + distinguish between forward_train (train and validate) and + forward_infer (decode). Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - Encoded output tensor - - Output lengths - - Optional next states for streaming inference + position embedded tensor and mask """ if self.training or not infer_mode: return self.forward_train(xs_pad, ilens, prev_states) @@ -206,22 +212,14 @@ def forward_train( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Process the input tensor for training or validation. - - This method applies the full contextual block transformer encoding process - to the input, suitable for training and validation phases. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Input tensor (B, L, D) - ilens (torch.Tensor): Input length (B) - prev_states (torch.Tensor, optional): Not used in training mode. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - Encoded output tensor - - Output lengths - - None (no states are returned in training mode) + position embedded tensor and mask """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) @@ -369,24 +367,14 @@ def forward_infer( prev_states: torch.Tensor = None, is_final: bool = True, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Process the input tensor for inference in streaming mode. - - This method applies the contextual block transformer encoding process - to the input, optimized for streaming inference. It handles partial inputs - and maintains state between calls for continuous processing. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Input tensor (B, L, D) - ilens (torch.Tensor): Input length (B) - prev_states (torch.Tensor, optional): Previous states from the last call. Defaults to None. - is_final (bool, optional): Whether this is the final block in the stream. Defaults to True. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - Encoded output tensor - - None (output lengths are not computed in inference mode) - - Next states to be used in the subsequent call, or None if is_final is True + position embedded tensor and mask """ if prev_states is None: prev_addin = None diff --git a/espnet2/asr/encoder/e_branchformer_encoder.py b/espnet2/asr/encoder/e_branchformer_encoder.py index abdd3cfbe2a..ae2381c234e 100644 --- a/espnet2/asr/encoder/e_branchformer_encoder.py +++ b/espnet2/asr/encoder/e_branchformer_encoder.py @@ -51,40 +51,16 @@ class EBranchformerEncoderLayer(torch.nn.Module): - """ - E-Branchformer encoder layer module. - - This class implements a single layer of the E-Branchformer encoder, which combines - self-attention and convolutional gating MLP branches. + """E-Branchformer encoder layer module. Args: - size (int): Model dimension. - attn (torch.nn.Module): Self-attention module. - cgmlp (torch.nn.Module): Convolutional Gating MLP module. - feed_forward (Optional[torch.nn.Module]): Feed-forward module. - feed_forward_macaron (Optional[torch.nn.Module]): Macaron-style feed-forward module. - dropout_rate (float): Dropout probability. - merge_conv_kernel (int): Kernel size of the depth-wise conv in merge module. Defaults to 3. - - Attributes: - size (int): Model dimension. - attn (torch.nn.Module): Self-attention module. - cgmlp (torch.nn.Module): Convolutional Gating MLP module. - feed_forward (Optional[torch.nn.Module]): Feed-forward module. - feed_forward_macaron (Optional[torch.nn.Module]): Macaron-style feed-forward module. - ff_scale (float): Scaling factor for feed-forward modules. - norm_ff (LayerNorm): Layer normalization for feed-forward module. - norm_ff_macaron (LayerNorm): Layer normalization for macaron-style feed-forward module. - norm_mha (LayerNorm): Layer normalization for multi-head attention module. - norm_mlp (LayerNorm): Layer normalization for MLP module. - norm_final (LayerNorm): Final layer normalization. - dropout (torch.nn.Dropout): Dropout layer. - depthwise_conv_fusion (torch.nn.Conv1d): Depth-wise convolution for branch fusion. - merge_proj (torch.nn.Linear): Linear projection for branch merging. - - Note: - This implementation is based on the paper "E-Branchformer: Branchformer with - Enhanced merging for speech recognition" by Kim et al. + size (int): model dimension + attn: standard self-attention or efficient attention + cgmlp: ConvolutionalGatingMLP + feed_forward: feed-forward module, optional + feed_forward: macaron-style feed-forward module, optional + dropout_rate (float): dropout probability + merge_conv_kernel (int): kernel size of the depth-wise conv in merge module """ def __init__( @@ -130,31 +106,17 @@ def __init__( self.merge_proj = torch.nn.Linear(size + size, size) def forward(self, x_input, mask, cache=None): - """ - Compute encoded features. - - This method processes the input through the E-Branchformer encoder layer, - including self-attention and convolutional gating MLP branches. + """Compute encoded features. Args: - x_input (Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]): Input tensor w/ or w/o positional embedding. + x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - w/o pos emb: Tensor (#batch, time, size). mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (Optional[torch.Tensor]): Cache tensor of the input (#batch, time - 1, size). Defaults to None. - + cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). Returns: - Union[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: Output tensor or tuple of tensors. - - If input includes positional embedding: ((#batch, time, size), (1, time, size)). - - If input doesn't include positional embedding: (#batch, time, size). - torch.Tensor: Updated mask tensor (#batch, time). - - Raises: - NotImplementedError: If cache is not None, which is currently not supported. - - Note: - The method applies various transformations including feed-forward layers, - self-attention, convolutional gating MLP, and branch merging. + torch.Tensor: Output tensor (#batch, time, size). + torch.Tensor: Mask tensor (#batch, time). """ if cache is not None: @@ -220,53 +182,7 @@ def forward(self, x_input, mask, cache=None): class EBranchformerEncoder(AbsEncoder): - """ - E-Branchformer encoder module. - - This class implements the E-Branchformer encoder, which combines self-attention - and convolutional gating MLP in a branched structure for speech recognition tasks. - - Args: - input_size (int): Dimension of input features. - output_size (int): Dimension of output features. Defaults to 256. - attention_heads (int): Number of attention heads. Defaults to 4. - attention_layer_type (str): Type of attention layer. Defaults to "rel_selfattn". - pos_enc_layer_type (str): Type of positional encoding layer. Defaults to "rel_pos". - rel_pos_type (str): Type of relative positional encoding. Defaults to "latest". - cgmlp_linear_units (int): Number of units in CGMLP linear layer. Defaults to 2048. - cgmlp_conv_kernel (int): Kernel size for CGMLP convolution. Defaults to 31. - use_linear_after_conv (bool): Whether to use linear layer after convolution. Defaults to False. - gate_activation (str): Activation function for the gate. Defaults to "identity". - num_blocks (int): Number of encoder blocks. Defaults to 12. - dropout_rate (float): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float): Dropout rate for positional encoding. Defaults to 0.1. - attention_dropout_rate (float): Dropout rate for attention. Defaults to 0.0. - input_layer (Optional[str]): Type of input layer. Defaults to "conv2d". - zero_triu (bool): Whether to zero out the upper triangular part of attention matrix. Defaults to False. - padding_idx (int): Padding index for embedding layer. Defaults to -1. - layer_drop_rate (float): Layer dropout rate. Defaults to 0.0. - max_pos_emb_len (int): Maximum length for positional embedding. Defaults to 5000. - use_ffn (bool): Whether to use feed-forward network. Defaults to False. - macaron_ffn (bool): Whether to use macaron-style feed-forward network. Defaults to False. - ffn_activation_type (str): Activation function for feed-forward network. Defaults to "swish". - linear_units (int): Number of units in feed-forward network. Defaults to 2048. - positionwise_layer_type (str): Type of position-wise layer. Defaults to "linear". - merge_conv_kernel (int): Kernel size for merge convolution. Defaults to 3. - interctc_layer_idx (Optional[List[int]]): Indices of layers to apply intermediate CTC. Defaults to None. - interctc_use_conditioning (bool): Whether to use intermediate CTC for conditioning. Defaults to False. - - Attributes: - embed (torch.nn.Module): Input embedding layer. - encoders (torch.nn.Module): Sequence of encoder layers. - after_norm (LayerNorm): Layer normalization after the encoder layers. - interctc_layer_idx (List[int]): Indices of layers to apply intermediate CTC. - interctc_use_conditioning (bool): Whether to use intermediate CTC for conditioning. - conditioning_layer (Optional[torch.nn.Module]): Conditioning layer for intermediate CTC. - - Note: - This implementation is based on the paper "E-Branchformer: Branchformer with - Enhanced merging for speech recognition" by Kim et al. - """ + """E-Branchformer encoder module.""" @typechecked def __init__( @@ -502,12 +418,6 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The dimension of the encoder output. - """ return self._output_size def forward( @@ -518,30 +428,18 @@ def forward( ctc: CTC = None, max_layer: int = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Calculate forward propagation. + """Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not used in current implementation. Defaults to None. - ctc (CTC): Intermediate CTC module. Defaults to None. - max_layer (int): Maximum number of layers to process. Defaults to None. - + prev_states (torch.Tensor): Not to be used now. + ctc (CTC): Intermediate CTC module. + max_layer (int): Layer depth below which InterCTC is applied. Returns: - Tuple[Union[torch.Tensor, Tuple[torch.Tensor, List[Tuple[int, torch.Tensor]]]], torch.Tensor, Optional[torch.Tensor]]: - - Output tensor (#batch, L, output_size) or tuple of output tensor and intermediate CTC outputs. - - Output length (#batch). - - None (placeholder for future use). - - Raises: - TooShortUttError: If the input is too short for subsampling. - - Note: - - If intermediate CTC is used, the output will be a tuple containing the final output - and a list of intermediate outputs with their corresponding layer indices. - - The third return value (None) is a placeholder for consistency with other encoders - and may be used in future implementations. + torch.Tensor: Output tensor (#batch, L, output_size). + torch.Tensor: Output length (#batch). + torch.Tensor: Not to be used now. """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/hubert_encoder.py b/espnet2/asr/encoder/hubert_encoder.py index 067294d7f53..b2d58c2074e 100644 --- a/espnet2/asr/encoder/hubert_encoder.py +++ b/espnet2/asr/encoder/hubert_encoder.py @@ -26,52 +26,63 @@ class TorchAudioHuBERTPretrainEncoder(AbsEncoder): - """ - TorchAudio Hubert encoder module for pretraining. - - This class implements the Hubert encoder using the TorchAudio implementation. - It can be used for pretraining Hubert models or fine-tuning for downstream tasks. + """Torch Audio Hubert encoder module. Args: - input_size (int, optional): Input feature dimension. Defaults to None. - extractor_mode (str, optional): Operation mode of feature extractor. Defaults to "group_norm". - extractor_conv_layer_config (List[List[int]], optional): Configuration of convolution layers in feature extractor. Defaults to a specific 7-layer configuration. - extractor_conv_bias (bool, optional): Whether to include bias in convolution operations. Defaults to False. - encoder_embed_dim (int, optional): Embedding dimension in encoder. Defaults to 768. - encoder_projection_dropout (float, optional): Dropout probability after input feature projection. Defaults to 0.1. - encoder_pos_conv_kernel (int, optional): Kernel size of convolutional positional embeddings. Defaults to 128. - encoder_pos_conv_groups (int, optional): Number of groups in convolutional positional embeddings. Defaults to 16. - encoder_num_layers (int, optional): Number of self-attention layers in transformer block. Defaults to 12. - encoder_num_heads (int, optional): Number of heads in self-attention layers. Defaults to 12. - encoder_attention_dropout (float, optional): Dropout probability in self-attention layer. Defaults to 0.1. - encoder_ff_interm_features (int, optional): Dimension of hidden features in feed-forward layer. Defaults to 3072. - encoder_ff_interm_dropout (float, optional): Dropout probability in feed-forward layer. Defaults to 0.0. - encoder_dropout (float, optional): Dropout probability at the end of feed-forward layer. Defaults to 0.1. - encoder_layer_norm_first (bool, optional): Controls the order of layer normalization. Defaults to False. - encoder_layer_drop (float, optional): Probability to drop each encoder layer during training. Defaults to 0.05. - mask_prob (float, optional): Probability for each token to be chosen as start of the span to be masked. Defaults to 0.8. - mask_selection (str, optional): How to choose the mask length. Defaults to "static". - mask_other (float, optional): Secondary mask argument. Defaults to 0.0. - mask_length (int, optional): The lengths of the mask. Defaults to 10. - no_mask_overlap (bool, optional): Whether to allow masks to overlap. Defaults to False. - mask_min_space (int, optional): Minimum space between spans if no overlap is enabled. Defaults to 1. - mask_channel_prob (float, optional): Probability of replacing a feature with 0. Defaults to 0.0. - mask_channel_selection (str, optional): How to choose the mask length for channel masking. Defaults to "static". - mask_channel_other (float, optional): Secondary mask argument for channel masking. Defaults to 0.0. - mask_channel_length (int, optional): Minimum space between spans for channel masking. Defaults to 10. - no_mask_channel_overlap (bool, optional): Whether to allow channel masks to overlap. Defaults to False. - mask_channel_min_space (int, optional): Minimum space between spans for channel masking if no overlap is enabled. Defaults to 1. - skip_masked (bool, optional): Whether to skip computing losses over masked frames. Defaults to False. - skip_nomask (bool, optional): Whether to skip computing losses over unmasked frames. Defaults to False. - num_classes (int, optional): The number of classes in the labels. Defaults to 100. - final_dim (int, optional): Project final representations and targets to this dimension. Defaults to 256. - feature_grad_mult (float, optional): The factor to scale gradients from the convolutional feature extraction layer. Defaults to 0.1. - finetuning (bool, optional): Whether to fine-tune the model with ASR or other tasks. Defaults to False. - freeze_encoder_updates (int, optional): The number of steps to freeze the encoder parameters in ASR fine-tuning. Defaults to 0. - - Note: - For more details on the Hubert model and its parameters, refer to the TorchAudio documentation: - https://pytorch.org/audio/stable/generated/torchaudio.models.hubert_pretrain_model.html + extractor_mode: Operation mode of feature extractor. + Valid values are "group_norm" or "layer_norm". + extractor_conv_layer_config: Configuration of convolution layers in feature + extractor. List of convolution configuration, + i.e. [[output_channel, kernel_size, stride], ...] + extractor_conv_bias: Whether to include bias term to each convolution + operation. + encoder_embed_dim: The dimension of embedding in encoder. + encoder_projection_dropout: The dropout probability applied after the input + feature is projected to "encoder_embed_dim". + encoder_pos_conv_kernel: Kernel size of convolutional positional embeddings. + encoder_pos_conv_groups: Number of groups of convolutional positional + embeddings. + encoder_num_layers: Number of self attention layers in transformer block. + encoder_num_heads: Number of heads in self attention layers. + encoder_attention_dropout: Dropout probability applied after softmax in + self-attention layer. + encoder_ff_interm_features: Dimension of hidden features in feed forward layer. + encoder_ff_interm_dropout: Dropout probability applied in feedforward layer. + encoder_dropout: Dropout probability applied at the end of feed forward layer. + encoder_layer_norm_first: Control the order of layer norm in transformer layer + and each encoder layer. If True, in transformer layer, layer norm is + applied before features are fed to encoder layers. + encoder_layer_drop: Probability to drop each encoder layer during training. + mask_prob: Probability for each token to be chosen as start of the span + to be masked. + mask_selection: How to choose the mask length. + Options: [static, uniform, normal, poisson]. + mask_other: Secondary mask argument (used for more complex distributions). + mask_length: The lengths of the mask. + no_mask_overlap: Whether to allow masks to overlap. + mask_min_space: Minimum space between spans (if no overlap is enabled). + mask_channel_prob: (float): The probability of replacing a feature with 0. + mask_channel_selection: How to choose the mask length for channel masking. + Options: [static, uniform, normal, poisson]. + mask_channel_other: Secondary mask argument for channel masking(used for more + complex distributions). + mask_channel_length: Minimum space between spans (if no overlap is enabled) + for channel masking. + no_mask_channel_overlap: Whether to allow channel masks to overlap. + mask_channel_min_space: Minimum space between spans for channel + masking(if no overlap is enabled). + skip_masked: If True, skip computing losses over masked frames. + skip_nomask: If True, skip computing losses over unmasked frames. + num_classes: The number of classes in the labels. + final_dim: Project final representations and targets to final_dim. + feature_grad_mult: The factor to scale the convolutional feature extraction + layer gradients by. The scale factor will not affect the forward pass. + finetuning: Whether to finetuning the model with ASR or other tasks. + freeze_encoder_updates: The number of steps to freeze the encoder parameters + in ASR finetuning. + Hubert specific Args: + Please refer to: + https://pytorch.org/audio/stable/generated/torchaudio.models.hubert_pretrain_model.html#torchaudio.models.hubert_pretrain_model """ @typechecked @@ -175,16 +186,6 @@ def __init__( self.freeze_encoder_updates = freeze_encoder_updates def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The dimension of the encoder's output. - - Note: - This method returns the value of the `_output_size` attribute, - which is typically set during the initialization of the encoder. - """ return self._output_size def forward( @@ -195,32 +196,14 @@ def forward( ys_pad_length: torch.Tensor = None, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the Hubert Pretrain Encoder. - - This method processes the input tensor through the encoder, applying different - forward passes based on whether the model is in pretraining, fine-tuning, or - evaluation mode. + """Forward Hubert Pretrain Encoder. Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - ys_pad (torch.Tensor, optional): Target tensor for pretraining. Defaults to None. - ys_pad_length (torch.Tensor, optional): Lengths of target tensor. Defaults to None. - prev_states (torch.Tensor, optional): Previous states. Not used in current implementation. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - Output tensor after forward pass. - - Output lengths. - - Optional additional information (e.g., feature penalty in pretraining). - - Note: - The behavior of this method differs based on the encoder's mode: - - In pretraining mode, it returns logit_m, logit_u, and feature_penalty. - - In fine-tuning mode, it applies masking and returns the encoded features. - - In evaluation mode, it processes the input without masking. + position embedded tensor and mask """ if not self.finetuning: @@ -293,65 +276,27 @@ def _eval_forward(self, xs_pad, ilens): return x, lengths, None def reload_pretrained_parameters(self): - """ - Reload the pretrained parameters of the Hubert model. - - This method reloads the pretrained parameters stored in the `pretrained_params` - attribute into the `hubert_pretrain_model`. It's useful for resetting the model - to its initial pretrained state, especially after fine-tuning or making changes - to the model parameters. - - Note: - This method uses `strict=False` when loading the state dict, which means - it will ignore any parameters in the pretrained state that don't match - the current model structure. This allows for flexibility in model architecture - changes while still loading compatible pretrained weights. - - Raises: - Any exceptions raised by `load_state_dict` method of PyTorch modules. - - Example: - >>> encoder = TorchAudioHuBERTPretrainEncoder() - >>> # After some training or parameter modifications - >>> encoder.reload_pretrained_parameters() - >>> print("Pretrained Hubert model parameters reloaded!") - """ + self.hubert_pretrain_model.load_state_dict(self.pretrained_params, strict=False) logging.info("Pretrained Hubert model parameters reloaded!") class FairseqHubertEncoder(AbsEncoder): - """ - FairSeq Hubert encoder module for loading pretrained weights and fine-tuning. - - This class implements the Hubert encoder using the FairSeq implementation. - It supports loading pretrained Hubert models and fine-tuning for downstream tasks. + """FairSeq Hubert encoder module, used for loading pretrained weight and finetuning Args: - input_size (int): Input feature dimension. - hubert_url (str, optional): URL to the Hubert pretrained model. Defaults to "./". - hubert_dir_path (str, optional): Directory to download the Hubert pretrained model. Defaults to "./". - output_size (int, optional): Dimension of the encoder output. Defaults to 256. - normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to False. - freeze_finetune_updates (int, optional): Number of updates to freeze all layers except the output layer. Defaults to 0. - dropout_rate (float, optional): Dropout rate. Defaults to 0.0. - activation_dropout (float, optional): Dropout rate in activation function. Defaults to 0.1. - attention_dropout (float, optional): Dropout rate in attention layers. Defaults to 0.0. - mask_length (int, optional): Length of the mask for Hubert. Defaults to 10. - mask_prob (float, optional): Probability of applying mask. Defaults to 0.75. - mask_selection (str, optional): Method of selecting masks. Defaults to "static". - mask_other (int, optional): Secondary mask argument. Defaults to 0. - apply_mask (bool, optional): Whether to apply masking during fine-tuning. Defaults to True. - mask_channel_length (int, optional): Length of the channel mask. Defaults to 64. - mask_channel_prob (float, optional): Probability of applying channel mask. Defaults to 0.5. - mask_channel_other (int, optional): Secondary channel mask argument. Defaults to 0. - mask_channel_selection (str, optional): Method of selecting channel masks. Defaults to "static". - layerdrop (float, optional): Probability of dropping a layer. Defaults to 0.1. - feature_grad_mult (float, optional): Multiplier for feature extractors gradient. Defaults to 0.0. - - Note: - This class requires FairSeq to be installed. It loads the Hubert model - using FairSeq's model loading utilities and supports various masking - and fine-tuning strategies specific to the Hubert architecture. + input_size: input dim + hubert_url: url to Hubert pretrained model + hubert_dir_path: directory to download the Wav2Vec2.0 pretrained model. + output_size: dimension of attention + normalize_before: whether to use layer_norm before the first block + freeze_finetune_updates: steps that freeze all layers except output layer + before tuning the whole model (nessasary to prevent overfit). + dropout_rate: dropout rate + activation_dropout: dropout rate in activation function + attention_dropout: dropout rate in attention + Hubert specific Args: + Please refer to: + https://github.com/pytorch/fairseq/blob/master/fairseq/models/hubert/hubert.py """ @typechecked @@ -487,17 +432,6 @@ def __init__( self.register_buffer("num_updates", torch.LongTensor([0])) def output_size(self) -> int: - """ - Get the output size of the FairSeq Hubert encoder. - - Returns: - int: The dimension of the encoder's output. - - Note: - This method returns the value of the `_output_size` attribute, - which is set during the initialization of the encoder. It represents - the dimension of the output features produced by the encoder. - """ return self._output_size def forward( @@ -506,31 +440,14 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the FairSeq Hubert ASR Encoder. - - This method processes the input tensor through the Hubert encoder, - applying masking if specified and handling the freezing of parameters - during the initial fine-tuning updates. + """Forward Hubert ASR Encoder. Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - xs_pad (torch.Tensor): Encoded features of shape (B, T, C), where T is the - output sequence length and C is the output dimension. - - olens (torch.Tensor): Output lengths of shape (B,). - - None: Placeholder for future extensions. - - Note: - - The method handles the gradual unfreezing of layers during fine-tuning. - - It applies masking based on the `apply_mask` attribute and the training state. - - The output may be further processed by an additional output layer if specified. - - Layer normalization may be applied before returning the output if `normalize_before` is True. + position embedded tensor and mask """ masks = make_pad_mask(ilens).to(xs_pad.device) @@ -569,60 +486,26 @@ def forward( return xs_pad, olens, None def reload_pretrained_parameters(self): - """ - Reload the pretrained parameters of the FairSeq Hubert encoder. - - This method reloads the pretrained parameters stored in the `pretrained_params` - attribute into the `encoders` module of the FairSeq Hubert encoder. It's useful - for resetting the model to its initial pretrained state, especially after - fine-tuning or making changes to the model parameters. - - Note: - This method uses `strict=False` when loading the state dict, which means - it will ignore any parameters in the pretrained state that don't match - the current model structure. This allows for flexibility in model architecture - changes while still loading compatible pretrained weights. - - Raises: - Any exceptions raised by `load_state_dict` method of PyTorch modules. - - Example: - >>> encoder = FairseqHubertEncoder(input_size=80) - >>> # After some training or parameter modifications - >>> encoder.reload_pretrained_parameters() - >>> print("Pretrained Hubert model parameters reloaded!") - """ + self.encoders.load_state_dict(self.pretrained_params, strict=False) logging.info("Pretrained Hubert model parameters reloaded!") class FairseqHubertPretrainEncoder(AbsEncoder): - """ - FairSeq Hubert pretrain encoder module, specifically designed for the pretraining stage. - - This class implements the Hubert encoder using the FairSeq implementation, tailored - for pretraining tasks. It sets up the Hubert model with specified configurations - and prepares it for pretraining on unlabeled data. + """FairSeq Hubert pretrain encoder module, only used for pretraining stage Args: - input_size (int, optional): Input feature dimension. Defaults to 1. - output_size (int, optional): Dimension of the encoder output. Defaults to 1024. - linear_units (int, optional): Dimension of feedforward layers. Defaults to 1024. - attention_heads (int, optional): Number of attention heads. Defaults to 12. - num_blocks (int, optional): Number of encoder blocks. Defaults to 12. - dropout_rate (float, optional): Dropout rate. Defaults to 0.0. - attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. - activation_dropout_rate (float, optional): Dropout rate for activation functions. Defaults to 0.0. - hubert_dict (str, optional): Path to the Hubert dictionary file. Defaults to "./dict.txt". - label_rate (int, optional): Label frame rate. Use -1 for sequence labels. Defaults to 100. - checkpoint_activations (bool, optional): Whether to use activation checkpointing. Defaults to False. - sample_rate (int, optional): Audio sample rate. Defaults to 16000. - use_amp (bool, optional): Whether to use automatic mixed precision. Defaults to False. - **kwargs: Additional keyword arguments for Hubert configuration. - - Note: - This class requires FairSeq to be installed. It sets up the Hubert model - using FairSeq's HubertConfig and HubertPretrainingConfig, and prepares - the model for pretraining tasks. + input_size: input dim + output_size: dimension of attention + linear_units: dimension of feedforward layers + attention_heads: the number of heads of multi head attention + num_blocks: the number of encoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + hubert_dict: target dictionary for Hubert pretraining + label_rate: label frame rate. -1 for sequence label + sample_rate: target sample rate. + use_amp: whether to use automatic mixed precision + normalize_before: whether to use layer_norm before the first block """ @typechecked @@ -701,17 +584,6 @@ def _build_dictionary(self, dictionary, hubert_dict_path): self.dictionaries = [dictionary] def output_size(self) -> int: - """ - Get the output size of the FairSeq Hubert pretrain encoder. - - Returns: - int: The dimension of the encoder's output. - - Note: - This method returns the value of the `_output_size` attribute, - which is set during the initialization of the encoder. It represents - the dimension of the output features produced by the Hubert pretrain encoder. - """ return self._output_size def forward( @@ -722,28 +594,14 @@ def forward( ys_pad_length: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the FairSeq Hubert Pretrain Encoder. - - This method processes the input tensor through the Hubert encoder for pretraining, - applying necessary masking and computing the pretraining objectives. + """Forward Hubert Pretrain Encoder. Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of shape (B,). - ys_pad (torch.Tensor): Target tensor for pretraining tasks. - ys_pad_length (torch.Tensor): Lengths of the target tensor. - prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing the encoder outputs. - The exact content of the tuple depends on the Hubert pretraining configuration. - - Note: - - This method first casts the mask embedding to the appropriate data type. - - It then applies masking to the input and processes it through the Hubert encoder. - - The method is specifically designed for the pretraining phase of Hubert. + position embedded tensor and mask """ self.cast_mask_emb() masks = make_pad_mask(ilens).to(xs_pad.device) @@ -758,48 +616,10 @@ def forward( return enc_outputs def cast_mask_emb(self): - """ - Cast the mask embedding to half precision if using automatic mixed precision. - - This method checks if automatic mixed precision (AMP) is enabled and if the - mask embedding is not already in half precision. If these conditions are met, - it casts the mask embedding to torch.cuda.HalfTensor. - - Note: - This method is typically called before the forward pass to ensure - that the mask embedding is in the correct precision for AMP computations. - It modifies the `mask_emb` parameter of the encoder in-place. - - Example: - >>> encoder = FairseqHubertPretrainEncoder(use_amp=True) - >>> encoder.cast_mask_emb() - # The mask_emb will be cast to half precision if it wasn't already - """ if self.use_amp and self.encoder.mask_emb.dtype != torch.cuda.HalfTensor: self.encoder.mask_emb = torch.nn.Parameter(self.encoder.mask_emb.half()) def reload_pretrained_parameters(self): - """ - Reinitialize the mask embedding of the Hubert encoder. - - This method reinitializes the mask embedding of the Hubert encoder with - random values drawn from a uniform distribution. The embedding is set to - half precision (float16) and its size is determined by the encoder's - configuration. - - Note: - - This method is typically used to reset the mask embedding to a new - random state, which can be useful when restarting pretraining or - when adapting a pretrained model to a new task. - - The method logs information about the reinitialization, including - the data type of the new mask embedding and whether automatic mixed - precision (AMP) is being used. - - Example: - >>> encoder = FairseqHubertPretrainEncoder() - >>> encoder.reload_pretrained_parameters() - # Logs: "Hubert mask embedding re-initialized!, torch.cuda.HalfTensor, False" - """ self.encoder.mask_emb = torch.nn.Parameter( torch.HalfTensor(self.cfg.encoder_embed_dim).uniform_() ) @@ -811,32 +631,7 @@ def reload_pretrained_parameters(self): def download_hubert(model_url, dir_path): - """ - Download the Hubert model from the given URL and save it to the specified directory. - - This function downloads the Hubert model if it doesn't exist in the specified directory. - It uses a file lock to ensure thread-safe downloading. - - Args: - model_url (str): The URL of the Hubert model to download. - dir_path (str): The directory path where the model should be saved. - - Returns: - str: The full path to the downloaded model file. - - Raises: - Any exceptions raised by os.makedirs, FileLock, or torch.hub.download_url_to_file. - - Example: - >>> model_url = "https://example.com/hubert_model.pt" - >>> dir_path = "./models" - >>> model_path = download_hubert(model_url, dir_path) - >>> print(f"Model downloaded to: {model_path}") - - Note: - This function uses FileLock to prevent multiple processes from downloading - the same model simultaneously. Make sure the FileLock library is installed. - """ + os.makedirs(dir_path, exist_ok=True) model_name = model_url.split("/")[-1] model_path = os.path.join(dir_path, model_name) diff --git a/espnet2/asr/encoder/hugging_face_transformers_encoder.py b/espnet2/asr/encoder/hugging_face_transformers_encoder.py index 90bc60ab6e1..1d363764ca3 100644 --- a/espnet2/asr/encoder/hugging_face_transformers_encoder.py +++ b/espnet2/asr/encoder/hugging_face_transformers_encoder.py @@ -23,36 +23,7 @@ class HuggingFaceTransformersEncoder(AbsEncoder): - """ - Hugging Face Transformers PostEncoder for ASR tasks. - - This class implements an encoder that utilizes pre-trained Hugging Face Transformer - models for automatic speech recognition (ASR) tasks. It can be used as a post-encoder - in ESPnet2 ASR systems. - - Attributes: - transformer (transformers.PreTrainedModel): The Transformer model used for encoding. - pretrained_params (dict): A copy of the initial pre-trained model parameters. - lang_token_id (int): The token ID for language, used if language-specific encoding is needed. - - Args: - input_size (int): The size of the input features. - model_name_or_path (str): The name or path of the pre-trained Hugging Face Transformer model. - lang_token_id (int, optional): The token ID for language. Defaults to -1 (not used). - - Raises: - ImportError: If the 'transformers' library is not installed. - - Note: - This class requires the 'transformers' library to be installed. If not available, - it will raise an ImportError with instructions on how to install it. - - Examples: - >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") - >>> input_tensor = torch.randn(32, 100, 80) # (batch_size, sequence_length, input_size) - >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) - >>> output, output_lengths = encoder(input_tensor, input_lengths) - """ + """Hugging Face Transformers PostEncoder.""" @typechecked def __init__( @@ -85,35 +56,7 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the Hugging Face Transformers encoder. - - This method processes the input tensor through the Transformer model and returns - the encoded representation along with the updated input lengths. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, sequence_length, input_size). - input_lengths (torch.Tensor): Tensor of input sequence lengths of shape (batch_size,). - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - output (torch.Tensor): Encoded representation from the Transformer model. - Shape: (batch_size, sequence_length, hidden_size). - - input_lengths (torch.Tensor): Updated input lengths, accounting for any - additional tokens (e.g., language token) added during processing. - - Note: - If a language token ID is specified (self.lang_token_id != -1), it will be - prepended to the input sequences, and the input lengths will be incremented accordingly. - - Examples: - >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") - >>> input_tensor = torch.randn(32, 100, 80) # (batch_size, sequence_length, input_size) - >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) - >>> output, output_lengths = encoder.forward(input_tensor, input_lengths) - >>> print(output.shape) # Expected: torch.Size([32, 100, 768]) - >>> print(output_lengths.shape) # Expected: torch.Size([32]) - """ + """Forward.""" args = {"return_dict": True} @@ -138,43 +81,11 @@ def forward( return output, input_lengths def reload_pretrained_parameters(self): - """ - Reload the pretrained parameters of the Transformer model. - - This method resets the Transformer model's parameters to their initial pretrained state. - It's useful for resetting the model to its original configuration after fine-tuning or - other modifications. - - Note: - This method uses the `pretrained_params` attribute, which is a deep copy of the - initial model parameters created during the initialization of the - HuggingFaceTransformersEncoder instance. - - Examples: - >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") - >>> # After some fine-tuning or parameter updates - >>> encoder.reload_pretrained_parameters() - >>> print("Pretrained Transformers model parameters reloaded!") - """ + self.transformer.load_state_dict(self.pretrained_params) logging.info("Pretrained Transformers model parameters reloaded!") def output_size(self) -> int: - """ - Get the output size of the Transformer encoder. - - This method returns the size of the hidden state output by the Transformer model. - - Returns: - int: The size of the hidden state (number of features) in the output of the Transformer model. - - Note: - The output size is determined by the `hidden_size` parameter in the Transformer model's configuration. - - Examples: - >>> encoder = HuggingFaceTransformersEncoder(input_size=80, model_name_or_path="bert-base-uncased") - >>> output_size = encoder.output_size() - >>> print(output_size) # Expected: 768 for bert-base-uncased - """ + """Get the output size.""" return self.transformer.config.hidden_size diff --git a/espnet2/asr/encoder/linear_encoder.py b/espnet2/asr/encoder/linear_encoder.py index 696871e0f9a..3cf2a607c3c 100644 --- a/espnet2/asr/encoder/linear_encoder.py +++ b/espnet2/asr/encoder/linear_encoder.py @@ -22,43 +22,16 @@ class LinearEncoder(AbsEncoder): - """ - Linear encoder module for processing input sequences. - - This class implements a linear encoder that can be used in various sequence processing tasks, - such as speech recognition or natural language processing. It supports different types of - input layers and can apply normalization and dropout to the input. - - Attributes: - embed: The input embedding layer, which can be one of several types (linear, conv2d, etc.). - normalize_before: A boolean indicating whether to apply layer normalization before processing. - after_norm: A LayerNorm layer applied after processing if normalize_before is True. + """Linear encoder module. Args: - input_size (int): The dimensionality of the input features. - output_size (int, optional): The dimensionality of the output features. Defaults to 256. - dropout_rate (float, optional): The dropout rate to apply. Defaults to 0.1. - input_layer (str, optional): The type of input layer to use. Can be 'linear', 'conv2d', - 'conv2d2', 'conv2d6', 'conv2d8', 'embed', or None. Defaults to 'conv2d'. - normalize_before (bool, optional): Whether to apply layer normalization before processing. - Defaults to True. - padding_idx (int, optional): The index used for padding in the embedding layer. - Only used when input_layer is 'embed'. Defaults to -1. - - Raises: - ValueError: If an unknown input_layer type is specified. - - Examples: - >>> encoder = LinearEncoder(input_size=80, output_size=256, input_layer='conv2d') - >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) - >>> input_lengths = torch.full((32,), 1000) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 250, 256]) # Time dimension is reduced due to conv2d subsampling - - Note: - The actual behavior of the encoder depends on the chosen input_layer type. - Some input layers (like conv2d variants) may modify the sequence length. + input_size: input dim + output_size: dimension of attention + linear_units: the number of units of position-wise feed forward + dropout_rate: dropout rate + input_layer: input layer type + normalize_before: whether to use layer_norm before the first block + padding_idx: padding_idx for input_layer=embed """ @typechecked @@ -106,17 +79,6 @@ def __init__( self.after_norm = LayerNorm(output_size) def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The dimensionality of the output features. - - Example: - >>> encoder = LinearEncoder(input_size=80, output_size=256) - >>> print(encoder.output_size()) - 256 - """ return self._output_size def forward( @@ -125,40 +87,14 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the LinearEncoder. - - This method processes the input tensor through the encoder, applying the specified - input layer, normalization, and any other transformations defined in the encoder. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input feature dimension. - ilens (torch.Tensor): Input lengths of each sequence in the batch, shape (B,). - prev_states (torch.Tensor, optional): Not used in this implementation. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - xs_pad (torch.Tensor): Encoded output tensor of shape (B, L', D'), - where L' is the potentially modified sequence length and D' is the output dimension. - - olens (torch.Tensor): Output lengths of each sequence in the batch, shape (B,). - - None: Placeholder for consistency with other encoder implementations. - - Raises: - TooShortUttError: If the input sequence is too short for subsampling operations - when using certain input layers (e.g., Conv2dSubsampling variants). - - Examples: - >>> encoder = LinearEncoder(input_size=80, output_size=256) - >>> xs_pad = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) - >>> ilens = torch.full((32,), 1000) - >>> output, olens, _ = encoder(xs_pad, ilens) - >>> print(output.shape, olens.shape) - torch.Size([32, 1000, 256]) torch.Size([32]) - - Note: - The actual output shape may vary depending on the input_layer type used in the encoder. - Some input layers (like conv2d variants) may reduce the sequence length. + position embedded tensor and mask """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/longformer_encoder.py b/espnet2/asr/encoder/longformer_encoder.py index 65c28446447..1de6c75c555 100644 --- a/espnet2/asr/encoder/longformer_encoder.py +++ b/espnet2/asr/encoder/longformer_encoder.py @@ -35,53 +35,46 @@ class LongformerEncoder(ConformerEncoder): - """ - Longformer SA Conformer encoder module. - - This class implements a Longformer Self-Attention Conformer encoder, which extends the - ConformerEncoder to handle longer sequences efficiently using the Longformer attention mechanism. + """Longformer SA Conformer encoder module. Args: input_size (int): Input dimension. - output_size (int): Dimension of attention. Defaults to 256. - attention_heads (int): The number of heads of multi head attention. Defaults to 4. - linear_units (int): The number of units of position-wise feed forward. Defaults to 2048. - num_blocks (int): The number of encoder blocks. Defaults to 6. - dropout_rate (float): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float): Dropout rate after adding positional encoding. Defaults to 0.1. - attention_dropout_rate (float): Dropout rate in attention. Defaults to 0.0. - input_layer (str): Input layer type. Defaults to "conv2d". - normalize_before (bool): Whether to use layer_norm before the first block. Defaults to True. - concat_after (bool): Whether to concat attention layer's input and output. Defaults to False. - positionwise_layer_type (str): Type of positionwise layer. Defaults to "linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. Defaults to 3. - macaron_style (bool): Whether to use macaron style for positionwise layer. Defaults to False. - rel_pos_type (str): Type of relative positional encoding. Defaults to "legacy". - pos_enc_layer_type (str): Encoder positional encoding layer type. Defaults to "abs_pos". - selfattention_layer_type (str): Encoder attention layer type. Defaults to "lf_selfattn". - activation_type (str): Encoder activation function type. Defaults to "swish". - use_cnn_module (bool): Whether to use convolution module. Defaults to True. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. Defaults to False. - cnn_module_kernel (int): Kernel size of convolution module. Defaults to 31. - padding_idx (int): Padding idx for input_layer=embed. Defaults to -1. - interctc_layer_idx (List[int]): Layer indices for intermediate CTC. Defaults to []. - interctc_use_conditioning (bool): Whether to use intermediate CTC predictions for conditioning. Defaults to False. - attention_windows (list): Layer-wise attention window sizes for longformer self-attn. Defaults to [100, 100, 100, 100, 100, 100]. - attention_dilation (list): Layer-wise attention dilation sizes for longformer self-attn. Defaults to [1, 1, 1, 1, 1, 1]. - attention_mode (str): Implementation for longformer self-attn. Defaults to "sliding_chunks". - - Attributes: - embed (torch.nn.Module): Input embedding layer. - encoders (torch.nn.Module): Encoder layers. - after_norm (torch.nn.Module): Layer normalization applied after the encoder layers. - interctc_layer_idx (List[int]): Layer indices for intermediate CTC. - interctc_use_conditioning (bool): Whether to use intermediate CTC predictions for conditioning. - conditioning_layer (torch.nn.Module): Conditioning layer for intermediate CTC. - - Note: - The Longformer attention mechanism allows for efficient processing of long sequences - by using a combination of local and global attention patterns. This implementation - supports different attention modes, window sizes, and dilation rates for each layer. + output_size (int): Dimension of attention. + attention_heads (int): The number of heads of multi head attention. + linear_units (int): The number of units of position-wise feed forward. + num_blocks (int): The number of decoder blocks. + dropout_rate (float): Dropout rate. + attention_dropout_rate (float): Dropout rate in attention. + positional_dropout_rate (float): Dropout rate after adding positional encoding. + input_layer (Union[str, torch.nn.Module]): Input layer type. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + If True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + If False, no additional linear will be applied. i.e. x -> x + att(x) + positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". + positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. + rel_pos_type (str): Whether to use the latest relative positional encoding or + the legacy one. The legacy relative positional encoding will be deprecated + in the future. More Details can be found in + https://github.com/espnet/espnet/pull/2816. + encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. + encoder_attn_layer_type (str): Encoder attention layer type. + activation_type (str): Encoder activation function type. + macaron_style (bool): Whether to use macaron style for positionwise layer. + use_cnn_module (bool): Whether to use convolution module. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. + cnn_module_kernel (int): Kernerl size of convolution module. + padding_idx (int): Padding idx for input_layer=embed. + attention_windows (list): Layer-wise attention window sizes + for longformer self-attn + attention_dilation(list): Layer-wise attention dilation sizes + for longformer self-attn + attention_mode(str): Implementation for longformer self-attn. + Default="sliding_chunks" + Choose 'n2', 'tvm' or 'sliding_chunks'. More details in + https://github.com/allenai/longformer + """ @typechecked @@ -293,12 +286,6 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: - """ - Returns the output size of the encoder. - - Returns: - int: The dimension of the encoder output. - """ return self._output_size def forward( @@ -309,30 +296,20 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Calculate forward propagation. + """Calculate forward propagation. Args: xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor, optional): Not used in current implementation. Defaults to None. - ctc (CTC, optional): CTC module for intermediate CTC loss. Defaults to None. - return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. + prev_states (torch.Tensor): Not to be used now. + ctc (CTC): ctc module for intermediate CTC loss + return_all_hs (bool): whether to return all hidden states Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - torch.Tensor: Output tensor (#batch, L, output_size). - - torch.Tensor: Output length (#batch). - - torch.Tensor: Not used in current implementation (always None). - - Raises: - TooShortUttError: If the input is too short for subsampling. - - Note: - This method applies the Longformer encoder to the input tensor, handling - padding, subsampling, and intermediate CTC calculations if specified. - It supports returning intermediate outputs for all encoder layers when - return_all_hs is True. + torch.Tensor: Output tensor (#batch, L, output_size). + torch.Tensor: Output length (#batch). + torch.Tensor: Not to be used now. + """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/rnn_encoder.py b/espnet2/asr/encoder/rnn_encoder.py index 439dcafacea..bc4912b5ec7 100644 --- a/espnet2/asr/encoder/rnn_encoder.py +++ b/espnet2/asr/encoder/rnn_encoder.py @@ -10,45 +10,17 @@ class RNNEncoder(AbsEncoder): - """ - RNN-based encoder for speech recognition tasks. - - This class implements a recurrent neural network (RNN) encoder, which can be - used as part of a speech recognition system. It supports both LSTM and GRU - cell types, with options for bidirectionality and projection layers. - - Attributes: - _output_size (int): The size of the output features. - rnn_type (str): The type of RNN cell used ('lstm' or 'gru'). - bidirectional (bool): Whether the RNN is bidirectional. - use_projection (bool): Whether to use projection layers. - enc (torch.nn.ModuleList): List containing the RNN module(s). + """RNNEncoder class. Args: - input_size (int): The number of expected features in the input. - rnn_type (str, optional): The type of RNN cell to use. Defaults to "lstm". - bidirectional (bool, optional): If True, becomes a bidirectional RNN. Defaults to True. - use_projection (bool, optional): Whether to use projection layers. Defaults to True. - num_layers (int, optional): Number of recurrent layers. Defaults to 4. - hidden_size (int, optional): The number of features in the hidden state. Defaults to 320. - output_size (int, optional): The number of output features. Defaults to 320. - dropout (float, optional): Dropout probability. Defaults to 0.0. - subsample (Optional[Sequence[int]], optional): Subsampling factors for each layer. - Defaults to (2, 2, 1, 1). + input_size: The number of expected features in the input + output_size: The number of output features + hidden_size: The number of hidden features + bidirectional: If ``True`` becomes a bidirectional LSTM + use_projection: Use projection layer or not + num_layers: Number of recurrent layers + dropout: dropout probability - Raises: - ValueError: If an unsupported rnn_type is provided. - - Example: - >>> input_size = 80 - >>> encoder = RNNEncoder(input_size, rnn_type="lstm", num_layers=3, hidden_size=256) - >>> input_tensor = torch.randn(32, 100, input_size) # (batch_size, sequence_length, input_size) - >>> input_lengths = torch.randint(50, 100, (32,)) # (batch_size,) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) - >>> print(output.shape) # Expected: torch.Size([32, 25, 320]) - - Note: - The actual output sequence length may be shorter than the input due to subsampling. """ @typechecked @@ -116,17 +88,6 @@ def __init__( ) def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The size of the output features from the encoder. - - Example: - >>> encoder = RNNEncoder(input_size=80, output_size=512) - >>> print(encoder.output_size()) - 512 - """ return self._output_size def forward( @@ -135,36 +96,6 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Forward pass of the RNN encoder. - - This method processes the input tensor through the RNN layers, applying - subsampling and masking as configured. - - Args: - xs_pad (torch.Tensor): Padded input tensor of shape (batch, time, feat). - ilens (torch.Tensor): Input lengths of each sequence in the batch. - prev_states (torch.Tensor, optional): Previous hidden states for incremental processing. - Defaults to None. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - - xs_pad (torch.Tensor): Output tensor after encoding and masking. - - ilens (torch.Tensor): Output lengths of each sequence in the batch. - - current_states (List[torch.Tensor]): List of current hidden states of the RNN. - - Example: - >>> encoder = RNNEncoder(input_size=80, hidden_size=320, output_size=320) - >>> xs_pad = torch.randn(2, 100, 80) # (batch_size, time, feat) - >>> ilens = torch.tensor([100, 80]) - >>> output, out_lens, states = encoder.forward(xs_pad, ilens) - >>> print(output.shape) # Expected: torch.Size([2, 25, 320]) - >>> print(out_lens) # Expected: tensor([25, 20]) - - Note: - The output tensor is masked based on the output lengths to ensure - padded regions are set to zero. - """ if prev_states is None: prev_states = [None] * len(self.enc) assert len(prev_states) == len(self.enc) diff --git a/espnet2/asr/encoder/transformer_encoder.py b/espnet2/asr/encoder/transformer_encoder.py index 9d9ac0853ea..ca42ede6359 100644 --- a/espnet2/asr/encoder/transformer_encoder.py +++ b/espnet2/asr/encoder/transformer_encoder.py @@ -36,43 +36,28 @@ class TransformerEncoder(AbsEncoder): - """ - Transformer encoder module. - - This class implements a Transformer encoder, which is a key component in sequence-to-sequence models. - It processes input sequences through multiple self-attention and feed-forward layers. + """Transformer encoder module. Args: - input_size (int): Dimensionality of the input features. - output_size (int, optional): Dimensionality of the output. Defaults to 256. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in position-wise feed-forward layers. Defaults to 2048. - num_blocks (int, optional): Number of encoder blocks. Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. Defaults to 0.1. - attention_dropout_rate (float, optional): Dropout rate in attention layers. Defaults to 0.0. - input_layer (str, optional): Type of input layer. Defaults to "conv2d". - pos_enc_class (class, optional): Positional encoding class. Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to apply layer normalization before the first block. Defaults to True. - concat_after (bool, optional): Whether to concatenate attention layer's input and output. Defaults to False. - positionwise_layer_type (str, optional): Type of position-wise layer. Defaults to "linear". - positionwise_conv_kernel_size (int, optional): Kernel size for position-wise conv1d layer. Defaults to 1. - padding_idx (int, optional): Padding index for embedding layer. Defaults to -1. - interctc_layer_idx (List[int], optional): Indices of layers for intermediate CTC. Defaults to []. - interctc_use_conditioning (bool, optional): Whether to use conditioning for intermediate CTC. Defaults to False. - layer_drop_rate (float, optional): Layer dropout rate. Defaults to 0.0. - - Attributes: - embed (torch.nn.Module): Input embedding layer. - encoders (torch.nn.Module): Stack of encoder layers. - after_norm (LayerNorm): Layer normalization applied after the final encoder layer. - interctc_layer_idx (List[int]): Indices of layers for intermediate CTC. - interctc_use_conditioning (bool): Whether to use conditioning for intermediate CTC. - conditioning_layer (torch.nn.Module): Conditioning layer for intermediate CTC. - - Note: - The encoder supports various types of input layers, including linear, convolutional, and embedding layers. - It also allows for intermediate CTC (Connectionist Temporal Classification) computations. + input_size: input dim + output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of decoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + positional_dropout_rate: dropout rate after adding positional encoding + input_layer: input layer type + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + positionwise_layer_type: linear of conv1d + positionwise_conv_kernel_size: kernel size of positionwise conv1d layer + padding_idx: padding_idx for input_layer=embed """ @typechecked @@ -187,16 +172,6 @@ def __init__( self.conditioning_layer = None def output_size(self) -> int: - """ - Returns the output size of the encoder. - - Returns: - int: The dimensionality of the encoder's output. - - Note: - This method returns the value of the `_output_size` attribute, which is set - during the initialization of the TransformerEncoder. - """ return self._output_size def forward( @@ -207,40 +182,17 @@ def forward( ctc: CTC = None, return_all_hs: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the TransformerEncoder. - - This method processes the input through the encoder layers and returns the encoded representation. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Padded input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of each sequence in the batch. - prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. - ctc (CTC, optional): CTC module for intermediate CTC loss. Defaults to None. - return_all_hs (bool, optional): Whether to return all hidden states. Defaults to False. + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + ctc (CTC): ctc module for intermediate CTC loss + return_all_hs (bool): whether to return all hidden states Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - xs_pad (torch.Tensor): Encoded output tensor of shape (B, L', D'), where L' is the - output sequence length and D' is the output dimension. - - olens (torch.Tensor): Output lengths of each sequence in the batch. - - None: Placeholder for future use (e.g., hidden states). - - Raises: - TooShortUttError: If the input sequence is too short for subsampling in the embedding layer. - - Note: - - The method applies input embedding, positional encoding, and multiple transformer encoder layers. - - If intermediate CTC layers are specified, it returns intermediate outputs as well. - - The output can be normalized if `normalize_before` is set to True during initialization. - - Examples: - >>> encoder = TransformerEncoder(input_size=80, output_size=256) - >>> xs_pad = torch.randn(2, 100, 80) # (batch_size, sequence_length, input_dim) - >>> ilens = torch.tensor([100, 80]) # Actual lengths of sequences in the batch - >>> output, olens, _ = encoder(xs_pad, ilens) - >>> print(output.shape) # Example output shape: torch.Size([2, 100, 256]) + position embedded tensor and mask """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/transformer_encoder_multispkr.py b/espnet2/asr/encoder/transformer_encoder_multispkr.py index cabcac823b0..8f79389a822 100644 --- a/espnet2/asr/encoder/transformer_encoder_multispkr.py +++ b/espnet2/asr/encoder/transformer_encoder_multispkr.py @@ -33,59 +33,30 @@ class TransformerEncoder(AbsEncoder): - """ - Transformer encoder module for speech recognition tasks. - - This class implements a Transformer-based encoder that can be used in various - speech recognition models. It supports different input layer types, positional - encodings, and customizable encoder architectures. - - Attributes: - _output_size (int): The output dimension of the encoder. - embed (torch.nn.Module): The input embedding layer. - normalize_before (bool): Whether to apply layer normalization before each block. - encoders (torch.nn.Module): The main encoder layers. - after_norm (torch.nn.Module): The final layer normalization (if normalize_before is True). - num_inf (int): The number of inference outputs. - encoders_sd (torch.nn.ModuleList): Speaker-dependent encoder layers. + """Transformer encoder module. Args: - input_size (int): Dimension of the input features. - output_size (int, optional): Dimension of the encoder output and attention. - Defaults to 256. - attention_heads (int, optional): Number of attention heads. Defaults to 4. - linear_units (int, optional): Number of units in position-wise feed-forward layers. - Defaults to 2048. - num_blocks (int, optional): Number of encoder blocks. Defaults to 6. - num_blocks_sd (int, optional): Number of speaker-dependent encoder blocks. - Defaults to 6. - dropout_rate (float, optional): Dropout rate. Defaults to 0.1. - positional_dropout_rate (float, optional): Dropout rate for positional encoding. - Defaults to 0.1. - attention_dropout_rate (float, optional): Dropout rate in attention layers. - Defaults to 0.0. - input_layer (str, optional): Type of input layer. Can be "conv2d", "linear", - "conv2d1", "conv2d2", "conv2d6", "conv2d8", "embed", or None. Defaults to "conv2d". - pos_enc_class (type, optional): Positional encoding class. - Defaults to PositionalEncoding. - normalize_before (bool, optional): Whether to use layer normalization before - each block. Defaults to True. - concat_after (bool, optional): Whether to concatenate attention layer's input - and output. Defaults to False. - positionwise_layer_type (str, optional): Type of position-wise layer. - Can be "linear", "conv1d", or "conv1d-linear". Defaults to "linear". - positionwise_conv_kernel_size (int, optional): Kernel size of position-wise - conv1d layer. Defaults to 1. - padding_idx (int, optional): Padding index for input_layer="embed". Defaults to -1. - num_inf (int, optional): Number of inference outputs. Defaults to 1. - - Raises: - ValueError: If an unknown input_layer type is specified. - NotImplementedError: If an unsupported positionwise_layer_type is specified. - - Note: - This implementation is based on the paper "Attention Is All You Need" - by Vaswani et al. (2017) and adapted for speech recognition tasks. + input_size: input dim + output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of recognition encoder blocks + num_blocks_sd: the number of speaker dependent encoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + positional_dropout_rate: dropout rate after adding positional encoding + input_layer: input layer type + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + positionwise_layer_type: linear of conv1d + positionwise_conv_kernel_size: kernel size of positionwise conv1d layer + padding_idx: padding_idx for input_layer=embed + num_inf: number of inference output """ @typechecked @@ -204,12 +175,6 @@ def __init__( ) def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The output dimension of the encoder. - """ return self._output_size def forward( @@ -218,35 +183,14 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the Transformer encoder. - - This method processes the input tensor through the encoder layers, applying - positional encoding, attention mechanisms, and feed-forward networks. + """Embed positions in tensor. Args: - xs_pad (torch.Tensor): Padded input tensor of shape (B, L, D), where B is - the batch size, L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of each sequence in the batch, shape (B,). - prev_states (torch.Tensor, optional): Not used in the current implementation. - Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - torch.Tensor: Encoded output tensor of shape (B, num_inf, L', D'), - where L' is the encoded sequence length and D' is the output dimension. - - torch.Tensor: Output lengths of each sequence in the batch after - encoding, shape (B, num_inf). - - None: Placeholder for future use, currently always None. - - Raises: - TooShortUttError: If the input sequence is too short for subsampling in - certain input layer types (Conv2dSubsampling variants). - - Note: - The method handles different input layer types and applies the appropriate - embedding and subsampling techniques before passing the data through the - encoder layers. + position embedded tensor and mask """ masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) diff --git a/espnet2/asr/encoder/vgg_rnn_encoder.py b/espnet2/asr/encoder/vgg_rnn_encoder.py index 28cb27f3db2..fd457e7f8ff 100644 --- a/espnet2/asr/encoder/vgg_rnn_encoder.py +++ b/espnet2/asr/encoder/vgg_rnn_encoder.py @@ -11,39 +11,17 @@ class VGGRNNEncoder(AbsEncoder): - """ - VGGRNNEncoder class for feature extraction in speech recognition tasks. - - This class combines a VGG (Visual Geometry Group) network with a Recurrent Neural Network (RNN) - to process input features for Automatic Speech Recognition (ASR) tasks. - - Attributes: - _output_size (int): The size of the output features. - rnn_type (str): The type of RNN used ('lstm' or 'gru'). - bidirectional (bool): Whether the RNN is bidirectional. - use_projection (bool): Whether to use projection layer after RNN. - enc (torch.nn.ModuleList): List of encoder modules (VGG2L and RNN/RNNP). + """VGGRNNEncoder class. Args: - input_size (int): The number of expected features in the input. - rnn_type (str, optional): Type of RNN to use. Defaults to "lstm". - bidirectional (bool, optional): If True, use bidirectional RNN. Defaults to True. - use_projection (bool, optional): Whether to use projection layer. Defaults to True. - num_layers (int, optional): Number of recurrent layers. Defaults to 4. - hidden_size (int, optional): The number of features in the hidden state. Defaults to 320. - output_size (int, optional): The number of output features. Defaults to 320. - dropout (float, optional): Dropout probability. Defaults to 0.0. - in_channel (int, optional): Number of input channels. Defaults to 1. + input_size: The number of expected features in the input + bidirectional: If ``True`` becomes a bidirectional LSTM + use_projection: Use projection layer or not + num_layers: Number of recurrent layers + hidden_size: The number of hidden features + output_size: The number of output features + dropout: dropout probability - Raises: - ValueError: If rnn_type is not 'lstm' or 'gru'. - - Example: - >>> input_size = 80 - >>> encoder = VGGRNNEncoder(input_size, rnn_type="lstm", num_layers=3) - >>> input_tensor = torch.randn(32, 100, input_size) # (batch, time, features) - >>> input_lengths = torch.randint(50, 100, (32,)) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) """ @typechecked @@ -102,17 +80,6 @@ def __init__( ) def output_size(self) -> int: - """ - Returns the output size of the encoder. - - Returns: - int: The number of output features from the encoder. - - Example: - >>> encoder = VGGRNNEncoder(input_size=80, output_size=320) - >>> print(encoder.output_size()) - 320 - """ return self._output_size def forward( @@ -121,33 +88,6 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Forward pass of the VGGRNNEncoder. - - This method processes the input through the VGG layers followed by RNN layers. - - Args: - xs_pad (torch.Tensor): Padded input tensor of shape (batch, time, feat). - ilens (torch.Tensor): Input lengths of each sequence in batch. - prev_states (torch.Tensor, optional): Previous states for RNN layers. Defaults to None. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - - xs_pad (torch.Tensor): Output tensor after encoding. - - ilens (torch.Tensor): Output lengths of each sequence in batch. - - current_states (List[torch.Tensor]): List of current states for RNN layers. - - Example: - >>> encoder = VGGRNNEncoder(input_size=80, output_size=320) - >>> xs_pad = torch.randn(32, 100, 80) # (batch, time, feat) - >>> ilens = torch.randint(50, 100, (32,)) - >>> output, output_lengths, states = encoder(xs_pad, ilens) - >>> print(output.shape) - torch.Size([32, 25, 320]) # (batch, reduced_time, output_size) - - Note: - The time dimension is typically reduced due to the VGG layers' stride operations. - """ if prev_states is None: prev_states = [None] * len(self.enc) assert len(prev_states) == len(self.enc) diff --git a/espnet2/asr/encoder/wav2vec2_encoder.py b/espnet2/asr/encoder/wav2vec2_encoder.py index 50711eba97b..8eb535dee3f 100644 --- a/espnet2/asr/encoder/wav2vec2_encoder.py +++ b/espnet2/asr/encoder/wav2vec2_encoder.py @@ -18,38 +18,16 @@ class FairSeqWav2Vec2Encoder(AbsEncoder): - """ - FairSeq Wav2Vec2 encoder module. - - This class implements an encoder based on the Wav2Vec2.0 model from FairSeq. - It can load pre-trained Wav2Vec2.0 models and use them as feature extractors - for speech recognition tasks. The encoder allows for fine-tuning of the - Wav2Vec2.0 model and provides options for output dimensionality adjustment. - - Attributes: - encoders (fairseq.models.wav2vec.wav2vec2.Wav2Vec2Model): The Wav2Vec2 model. - output_layer (torch.nn.Sequential): Optional layer for output dimension adjustment. - after_norm (espnet.nets.pytorch_backend.transformer.layer_norm.LayerNorm): Optional layer normalization. + """FairSeq Wav2Vec2 encoder module. Args: - input_size (int): Input dimension (not used in the current implementation). - w2v_url (str): URL to the Wav2Vec2.0 pretrained model. - w2v_dir_path (str, optional): Directory to download the Wav2Vec2.0 pretrained model. Defaults to "./". - output_size (int, optional): Dimension of the output. Defaults to 256. - normalize_before (bool, optional): Whether to use layer normalization before the first block. Defaults to False. - freeze_finetune_updates (int, optional): Number of updates to freeze the model before fine-tuning. Defaults to 0. - - Note: - This class requires FairSeq to be installed. If not installed, it will raise an ImportError - with instructions on how to install FairSeq. - - Examples: - >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") - >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) - >>> input_lengths = torch.full((32,), 1000) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 1000, 256]) + input_size: input dim + output_size: dimension of attention + w2v_url: url to Wav2Vec2.0 pretrained model + w2v_dir_path: directory to download the Wav2Vec2.0 pretrained model. + normalize_before: whether to use layer_norm before the first block + finetune_last_n_layers: last n layers to be finetuned in Wav2Vec2.0 + 0 means to finetune every layer if freeze_w2v=False. """ @typechecked @@ -115,21 +93,6 @@ def __init__( self.register_buffer("num_updates", torch.LongTensor([0])) def output_size(self) -> int: - """ - Get the output size of the encoder. - - Returns: - int: The size of the output tensor along the feature dimension. - - Note: - This method returns the value of the `_output_size` attribute, - which is set during the initialization of the encoder. - - Examples: - >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt", output_size=512) - >>> print(encoder.output_size()) - 512 - """ return self._output_size def forward( @@ -138,38 +101,14 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the FairSeqWav2Vec2 Encoder. - - This method processes the input tensor through the Wav2Vec2 model and optional output layer. + """Forward FairSeqWav2Vec2 Encoder. Args: - xs_pad (torch.Tensor): Input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the input dimension. - ilens (torch.Tensor): Input lengths of shape (B,) representing the valid length - of each sequence in the batch. - prev_states (torch.Tensor, optional): Not used in the current implementation. Defaults to None. - + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - torch.Tensor: Output tensor of shape (B, T, C), where T is the output sequence length - and C is the output dimension. - - torch.Tensor: Output lengths of shape (B,) representing the valid length of each - output sequence in the batch. - - Optional[torch.Tensor]: Always None in the current implementation. - - Note: - - The method handles the freezing and unfreezing of parameters based on the - `freeze_finetune_updates` attribute. - - If `normalize_before` is True, layer normalization is applied to the output. - - Examples: - >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") - >>> input_tensor = torch.randn(32, 1000, 80) # (batch_size, time_steps, features) - >>> input_lengths = torch.full((32,), 1000) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) - >>> print(output.shape, output_lengths.shape) - torch.Size([32, 1000, 256]) torch.Size([32]) + position embedded tensor and mask """ masks = make_pad_mask(ilens).to(xs_pad.device) @@ -205,56 +144,12 @@ def forward( return xs_pad, olens, None def reload_pretrained_parameters(self): - """ - Reload the pretrained parameters of the Wav2Vec model. - - This method resets the encoder's parameters to their original pretrained values. - It's useful when you want to start from the initial pretrained state after - fine-tuning or other modifications to the model. - - Note: - This method uses the `pretrained_params` attribute, which is a deep copy - of the initial model state dictionary created during the encoder's initialization. - - Examples: - >>> encoder = FairSeqWav2Vec2Encoder(input_size=80, w2v_url="https://example.com/wav2vec_model.pt") - >>> # After some fine-tuning or parameter updates - >>> encoder.reload_pretrained_parameters() - >>> print("Pretrained Wav2Vec model parameters reloaded!") - """ + self.encoders.load_state_dict(self.pretrained_params) logging.info("Pretrained Wav2Vec model parameters reloaded!") def download_w2v(model_url, dir_path): - """ - Download the Wav2Vec model and its dictionary. - - This function downloads the Wav2Vec model and its corresponding dictionary - if they don't already exist in the specified directory. It uses FileLock - to ensure thread-safe downloads. - - Args: - model_url (str): The URL of the Wav2Vec model to download. - dir_path (str): The directory path where the model and dictionary - should be saved. - - Returns: - str: The local file path of the downloaded Wav2Vec model. - - Raises: - OSError: If there are issues creating the directory or downloading files. - - Examples: - >>> model_url = "https://example.com/wav2vec_model.pt" - >>> dir_path = "./models" - >>> local_model_path = download_w2v(model_url, dir_path) - >>> print(local_model_path) - './models/wav2vec_model.pt' - - Note: - This function also downloads a dictionary file from a hardcoded URL: - 'https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt' - """ + os.makedirs(dir_path, exist_ok=True) model_name = model_url.split("/")[-1] model_path = os.path.join(dir_path, model_name) diff --git a/espnet2/asr/encoder/whisper_encoder.py b/espnet2/asr/encoder/whisper_encoder.py index f8c3e806d2f..5e96b9b8900 100644 --- a/espnet2/asr/encoder/whisper_encoder.py +++ b/espnet2/asr/encoder/whisper_encoder.py @@ -10,46 +10,9 @@ class OpenAIWhisperEncoder(AbsEncoder): - """ - OpenAI Whisper-based Speech Encoder for ASR tasks. - - This class implements a speech encoder based on OpenAI's Whisper model, - designed for Automatic Speech Recognition (ASR) tasks. It uses a Transformer - architecture and can be initialized with various Whisper model sizes. - - Attributes: - n_fft (int): Number of FFT points. - win_length (int): Window length for STFT. - hop_length (int): Hop length for STFT. - n_mels (int): Number of mel filterbanks. - mel_filters (function): Function to create mel filterbanks. - dropout (torch.nn.Dropout): Dropout layer. - encoders (whisper.model.Encoder): Whisper encoder layers. - specaug (SpecAug): SpecAugment layer for data augmentation. - do_pad_trim (bool): Whether to pad or trim input audio. - pad_samples (int): Number of samples to pad to. - - Args: - input_size (int): Input feature size. Defaults to 1. - dropout_rate (float): Dropout rate. Defaults to 0.0. - whisper_model (str): Whisper model size to use. Defaults to "small". - download_dir (Optional[str]): Directory to download Whisper model. Defaults to None. - use_specaug (bool): Whether to use SpecAugment. Defaults to False. - specaug_conf (Union[dict, None]): SpecAugment configuration. Defaults to None. - do_pad_trim (bool): Whether to pad or trim input audio. Defaults to False. - - Raises: - ImportError: If the whisper package is not properly installed. - - Note: - This encoder requires the `whisper` package to be installed. - It can be installed using the provided installation script. - - Example: - >>> encoder = OpenAIWhisperEncoder(whisper_model="base", use_specaug=True) - >>> input_tensor = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> output, output_lengths, _ = encoder(input_tensor, input_lengths) + """Transformer-based Speech Encoder from OpenAI's Whisper Model: + + URL: https://github.com/openai/whisper """ @typechecked @@ -104,21 +67,6 @@ def __init__( self.pad_samples = N_SAMPLES def output_size(self) -> int: - """ - Returns the output size of the encoder. - - This method returns the dimensionality of the encoder's output features. - - Returns: - int: The size of the output feature vector, which corresponds to the - number of units in the final layer normalization of the encoder. - - Example: - >>> encoder = OpenAIWhisperEncoder(whisper_model="base") - >>> output_dim = encoder.output_size() - >>> print(output_dim) - 512 # This may vary depending on the Whisper model size - """ return self.encoders.ln_post.normalized_shape[-1] def pad_or_trim( @@ -127,33 +75,9 @@ def pad_or_trim( length: int, axis: int = -1, ) -> torch.Tensor: - """ - Pad or trim the input tensor to a specified length along a given axis. - - This method is used to ensure that the input tensor has a consistent length, - which is particularly useful for zero-shot inference cases. - - Args: - array (torch.Tensor): The input tensor to be padded or trimmed. - length (int): The desired length of the tensor along the specified axis. - axis (int, optional): The axis along which to pad or trim. Defaults to -1 (last dimension). + """Pad or trim the audio array to N_SAMPLES. - Returns: - torch.Tensor: The padded or trimmed tensor with the specified length along the given axis. - - Raises: - ValueError: If the input tensor has fewer dimensions than the specified axis. - - Example: - >>> encoder = OpenAIWhisperEncoder() - >>> input_tensor = torch.randn(1, 12000) - >>> padded_tensor = encoder.pad_or_trim(input_tensor, length=16000) - >>> print(padded_tensor.shape) - torch.Size([1, 16000]) - - Note: - If the input tensor is longer than the specified length, it will be trimmed. - If it's shorter, it will be padded with zeros. + Used in zero-shot inference cases. """ if array.shape[axis] > length: array = array.index_select( @@ -172,36 +96,7 @@ def log_mel_spectrogram( audio: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: - """ - Compute the log-mel spectrogram of the input audio. - - This method implements the log-mel spectrogram computation native to Whisper training. - It performs short-time Fourier transform (STFT) on the input audio, applies mel filters, - and computes the log of the resulting spectrogram. - - Args: - audio (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). - ilens (torch.Tensor, optional): Tensor containing the lengths of each audio in the batch. - Defaults to None. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - log_spec (torch.Tensor): Log-mel spectrogram of shape (batch_size, n_mels, time). - - olens (Optional[torch.Tensor]): Tensor containing the lengths of each spectrogram - in the batch. Returns None if ilens is None. - - Note: - - The method uses the Whisper-specific parameters for STFT and mel filterbank. - - The last frame of the STFT is discarded to match Whisper's behavior. - - The log spectrogram is clamped and normalized as per Whisper's preprocessing. - - Example: - >>> encoder = OpenAIWhisperEncoder() - >>> audio = torch.randn(2, 16000) # 2 audio samples of 1 second each at 16kHz - >>> log_spec, spec_lengths = encoder.log_mel_spectrogram(audio) - >>> print(log_spec.shape) - torch.Size([2, 80, 100]) # (batch_size, n_mels, time) - """ + """Use log-mel spectrogram computation native to Whisper training""" window = torch.hann_window(self.win_length).to(audio.device) stft = torch.stft( audio, self.n_fft, self.hop_length, window=window, return_complex=True @@ -233,37 +128,6 @@ def whisper_encode( input: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: - """ - Encode input features using the Whisper encoder. - - This method applies the Whisper encoder to the input features, including - convolutional layers, positional embedding, and transformer blocks. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, n_mels, time). - ilens (torch.Tensor, optional): Tensor containing the lengths of each input - in the batch. Defaults to None. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - x (torch.Tensor): Encoded output of shape (batch_size, time, d_model). - - olens (Optional[torch.Tensor]): Tensor containing the lengths of each - encoded output in the batch. Returns None if ilens is None. - - Note: - - The method applies two convolutional layers followed by transformer blocks. - - Positional embedding is added to the output of convolutional layers. - - Due to positional encoding limitations, audios longer than 30 seconds - may not be fully encoded. - - Dropout is applied between transformer blocks during training. - - Example: - >>> encoder = OpenAIWhisperEncoder() - >>> input_features = torch.randn(2, 80, 100) # (batch_size, n_mels, time) - >>> encoded_output, output_lengths = encoder.whisper_encode(input_features) - >>> print(encoded_output.shape) - torch.Size([2, 100, 512]) # (batch_size, time, d_model) - """ x = F.gelu(self.encoders.conv1(input)) x = F.gelu(self.encoders.conv2(x)) x = x.permute(0, 2, 1) @@ -307,39 +171,6 @@ def forward( ilens: torch.Tensor, prev_states: torch.Tensor = None, ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """ - Forward pass of the OpenAI Whisper Encoder. - - This method processes the input audio through the entire encoder pipeline, - including optional padding/trimming, log-mel spectrogram computation, - optional SpecAugment, and Whisper encoding. - - Args: - xs_pad (torch.Tensor): Padded input tensor of shape (batch_size, T). - ilens (torch.Tensor): Tensor of input lengths of shape (batch_size,). - prev_states (torch.Tensor, optional): Tensor containing previous states. - Not used in this implementation. Defaults to None. - - Returns: - Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - xs_pad (torch.Tensor): Encoded output of shape (batch_size, T', D). - - olens (torch.Tensor): Tensor of output lengths of shape (batch_size,). - - None: Placeholder for consistency with AbsEncoder interface. - - Note: - - If `do_pad_trim` is True, input will be padded or trimmed to `pad_samples`. - - SpecAugment is applied during training if `specaug` is not None. - - The method handles the entire encoding process from raw audio to - final encoded representations. - - Example: - >>> encoder = OpenAIWhisperEncoder(do_pad_trim=True, use_specaug=True) - >>> input_audio = torch.randn(2, 16000) # 2 audio samples of 1 second each at 16kHz - >>> input_lengths = torch.tensor([16000, 16000]) - >>> output, output_lengths, _ = encoder(input_audio, input_lengths) - >>> print(output.shape) - torch.Size([2, 100, 512]) # (batch_size, time, d_model) - """ if self.do_pad_trim: xs_pad = self.pad_or_trim(xs_pad, self.pad_samples) diff --git a/espnet2/asr/espnet_model.py b/espnet2/asr/espnet_model.py index dd2a7c30dee..b5224585cf7 100644 --- a/espnet2/asr/espnet_model.py +++ b/espnet2/asr/espnet_model.py @@ -35,35 +35,7 @@ def autocast(enabled=True): class ESPnetASRModel(AbsESPnetModel): - """ - CTC-attention hybrid Encoder-Decoder model for Automatic Speech Recognition (ASR). - - This class implements a hybrid model that combines CTC (Connectionist Temporal Classification) - and attention-based approaches for ASR. It supports both regular attention decoder and - transducer decoder architectures. - - The model consists of several components: - - Frontend processing (optional) - - SpecAugment for data augmentation (optional) - - Normalization (optional) - - Pre-encoder (optional) - - Encoder - - Post-encoder (optional) - - Decoder (attention-based or transducer-based) - - CTC module - - Joint network (for transducer architecture) - - It can be used for various ASR tasks and supports features such as: - - CTC/Attention multi-task learning - - Intermediate CTC - - Transducer decoding - - Multi-blank transducer - - Auxiliary CTC tasks - - Multilingual ASR with language token - - The model calculates losses for CTC, attention, and/or transducer components, - and provides error rates (CER/WER) during evaluation. - """ + """CTC-attention hybrid Encoder-Decoder model""" @typechecked def __init__( @@ -237,33 +209,14 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """ - Frontend + Encoder + Decoder + Calc loss + """Frontend + Encoder + Decoder + Calc loss Args: - speech (torch.Tensor): Input speech tensor (Batch, Length, ...). - speech_lengths (torch.Tensor): Speech length tensor (Batch, ). - text (torch.Tensor): Text tensor (Batch, Length). - text_lengths (torch.Tensor): Text length tensor (Batch,). - kwargs: Additional keyword arguments. "utt_id" is among the input. - - Returns: - Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - - Loss tensor - - Dictionary containing various statistics and metrics: - - loss_ctc: CTC loss (if applicable) - - loss_att: Attention loss (if applicable) - - loss_transducer: Transducer loss (if applicable) - - acc: Attention accuracy - - cer: Character Error Rate - - wer: Word Error Rate - - loss: Total loss - - Batch size as a weight tensor - - Note: - The method expects input tensors to have consistent batch sizes across all inputs. - It supports CTC, attention, and transducer-based loss calculations depending on the model configuration. - Intermediate CTC loss is also calculated if the interctc_weight is non-zero. + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + kwargs: "utt_id" is among the input. """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -408,56 +361,17 @@ def collect_feats( text_lengths: torch.Tensor, **kwargs, ) -> Dict[str, torch.Tensor]: - """ - Collect features for the input speech. - - This method extracts features from the input speech using the model's feature extractor. - - Args: - speech (torch.Tensor): Input speech tensor (Batch, Length, ...). - speech_lengths (torch.Tensor): Speech length tensor (Batch, ). - text (torch.Tensor): Text tensor (Batch, Length). - text_lengths (torch.Tensor): Text length tensor (Batch,). - kwargs: Additional keyword arguments. - - Returns: - Dict[str, torch.Tensor]: A dictionary containing: - - "feats": Extracted features tensor. - - "feats_lengths": Lengths of the extracted features. - - Note: - This method is typically used for feature extraction during data preparation or analysis. - It does not perform any encoding or decoding operations. - """ feats, feats_lengths = self._extract_feats(speech, speech_lengths) return {"feats": feats, "feats_lengths": feats_lengths} def encode( self, speech: torch.Tensor, speech_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Encode speech features into a latent representation. - - This method performs the following steps: - 1. Extract features from the input speech - 2. Apply data augmentation (if SpecAugment is enabled and in training mode) - 3. Normalize the features (if normalization is enabled) - 4. Apply pre-encoding (if a pre-encoder is defined) - 5. Encode the features using the main encoder - 6. Apply post-encoding (if a post-encoder is defined) + """Frontend + Encoder. Note that this method is used by asr_inference.py Args: - speech (torch.Tensor): Input speech tensor (Batch, Length, ...). - speech_lengths (torch.Tensor): Speech length tensor (Batch, ). - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - Encoded speech tensor (Batch, Length2, Dim2). - - Encoded speech lengths tensor (Batch, ). - - Note: - This method is used by asr_inference.py and is a key component of the ASR pipeline. - It handles the entire encoding process from raw input to encoder output. + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) """ with autocast(False): # 1. Extract feats @@ -541,25 +455,15 @@ def nll( ys_pad: torch.Tensor, ys_pad_lens: torch.Tensor, ) -> torch.Tensor: - """ - Compute negative log likelihood (nll) from transformer-decoder. + """Compute negative log likelihood(nll) from transformer-decoder - This method calculates the negative log likelihood for the given encoder output - and target sequences using the transformer decoder. + Normally, this function is called in batchify_nll. Args: - encoder_out (torch.Tensor): Encoder output tensor (Batch, Length, Dim). - encoder_out_lens (torch.Tensor): Encoder output length tensor (Batch,). - ys_pad (torch.Tensor): Padded target sequence tensor (Batch, Length). - ys_pad_lens (torch.Tensor): Target sequence length tensor (Batch,). - - Returns: - torch.Tensor: Negative log likelihood tensor (Batch,). - - Note: - This function is typically called within the batchify_nll method. - It adds start-of-sequence (sos) and end-of-sequence (eos) tokens to the target sequence, - forwards the input through the decoder, and computes the cross-entropy loss. + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) """ ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) ys_in_lens = ys_pad_lens + 1 @@ -590,27 +494,18 @@ def batchify_nll( ys_pad_lens: torch.Tensor, batch_size: int = 100, ): - """ - Compute negative log likelihood (nll) from transformer-decoder in batches. - - This method calculates the negative log likelihood for the given encoder output - and target sequences using the transformer decoder, processing the input in batches - to avoid potential out-of-memory (OOM) issues. + """Compute negative log likelihood(nll) from transformer-decoder + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. Args: - encoder_out (torch.Tensor): Encoder output tensor (Batch, Length, Dim). - encoder_out_lens (torch.Tensor): Encoder output length tensor (Batch,). - ys_pad (torch.Tensor): Padded target sequence tensor (Batch, Length). - ys_pad_lens (torch.Tensor): Target sequence length tensor (Batch,). - batch_size (int, optional): Number of samples to process in each batch. Defaults to 100. - - Returns: - torch.Tensor: Negative log likelihood tensor (Batch,). - - Note: - This method is designed to handle large inputs by splitting them into smaller batches. - It calls the nll method for each batch and combines the results. - Adjust the batch_size parameter to optimize memory usage and processing speed. + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + GPU memory usage """ total_num = encoder_out.size(0) if total_num <= batch_size: diff --git a/espnet2/asr/frontend/abs_frontend.py b/espnet2/asr/frontend/abs_frontend.py index 5fa4c63504d..8f785e38d9e 100644 --- a/espnet2/asr/frontend/abs_frontend.py +++ b/espnet2/asr/frontend/abs_frontend.py @@ -5,86 +5,12 @@ class AbsFrontend(torch.nn.Module, ABC): - """ - Abstract base class for frontend modules in a neural network. - - This class defines the interface for frontend modules, which are typically - used in speech processing tasks to convert raw input signals into - feature representations. - - Attributes: - None - - Examples: - >>> class MyFrontend(AbsFrontend): - ... def output_size(self) -> int: - ... return 80 - ... def forward(self, input: torch.Tensor, input_lengths: torch.Tensor): - ... # Implementation here - ... return features, feature_lengths - - Note: - Subclasses must implement the `output_size` and `forward` methods. - """ - @abstractmethod def output_size(self) -> int: - """ - Returns the output size of the frontend module. - - This method should be implemented by subclasses to specify the - dimensionality of the feature representation produced by the frontend. - - Returns: - int: The size of the output feature dimension. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Examples: - >>> class MyFrontend(AbsFrontend): - ... def output_size(self) -> int: - ... return 80 - >>> frontend = MyFrontend() - >>> print(frontend.output_size()) - 80 - """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Processes the input and returns the feature representation. - - This method should be implemented by subclasses to define how the input - is transformed into features. - - Args: - input (torch.Tensor): The input tensor to be processed. - input_lengths (torch.Tensor): The lengths of each sequence in the input batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - features (torch.Tensor): The processed features. - - feature_lengths (torch.Tensor): The lengths of each sequence in the feature batch. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Examples: - >>> class MyFrontend(AbsFrontend): - ... def forward(self, input: torch.Tensor, input_lengths: torch.Tensor): - ... # Assume some processing here - ... features = input * 2 - ... feature_lengths = input_lengths - ... return features, feature_lengths - >>> frontend = MyFrontend() - >>> input_tensor = torch.randn(32, 1000) # Batch of 32, sequence length 1000 - >>> input_lengths = torch.full((32,), 1000) - >>> features, feature_lengths = frontend(input_tensor, input_lengths) - >>> features.shape - torch.Size([32, 1000]) - """ raise NotImplementedError diff --git a/espnet2/asr/frontend/asteroid_frontend.py b/espnet2/asr/frontend/asteroid_frontend.py index cab504e964a..4e9237081ae 100644 --- a/espnet2/asr/frontend/asteroid_frontend.py +++ b/espnet2/asr/frontend/asteroid_frontend.py @@ -16,38 +16,18 @@ class AsteroidFrontend(AbsFrontend): - """ - Asteroid Filterbank Frontend for audio feature extraction. - - This class implements a Sinc-convolutional-based audio feature extractor using the - Asteroid filterbank. It provides functionality similar to using a sliding window - frontend with a sinc preencoder. - - The frontend applies preemphasis, normalization, and frame-wise feature extraction - using parameterized analytic filterbanks as described in Pariente et al. (2020). - - Attributes: - sinc_filters (int): Number of filters for the sinc convolution. - sinc_kernel_size (int): Kernel size for the sinc convolution. - sinc_stride (int): Stride size for the first sinc-conv layer, determining - the compression rate (Hz). - output_dim (int): Output dimension of the feature extraction. - - Note: - This frontend is primarily used in sentence-level classification tasks - (e.g., speaker recognition). Its effectiveness in other applications - has not been fully investigated. - - Example: - >>> frontend = AsteroidFrontend(sinc_filters=256, sinc_kernel_size=251, sinc_stride=16) - >>> input_tensor = torch.randn(32, 16000) # (batch_size, time) - >>> input_lengths = torch.full((32,), 16000) - >>> output, output_lengths = frontend(input_tensor, input_lengths) - >>> print(output.shape) # (batch_size, time', features) - - References: - M. Pariente, S. Cornell, A. Deleforge and E. Vincent, - "Filterbank design for end-to-end speech separation," in Proc. ICASSP, 2020 + """Asteroid Filterbank Frontend. + + Provides a Sinc-convolutional-based audio feature extractor. The same + function can be achieved by using `sliding_winodw frontend + + sinc preencoder`. + + NOTE(jiatong): this function is used in sentence-level classification + tasks (e.g., spk). Other usages are not fully investigated. + + NOTE(jeeweon): this function implements the parameterized analytic + filterbank layer in M. Pariente, S. Cornell, A. Deleforge and E. Vincent, + "Filterbank design for end-to-end speech separation," in Proc. ICASSP, 2020 """ @typechecked @@ -92,38 +72,14 @@ def __init__( def forward( self, input: torch.Tensor, input_length: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Apply the Asteroid filterbank frontend to the input audio. - - This method processes the input audio through the Asteroid filterbank, - applying preemphasis, normalization, and frame-wise feature extraction. + """Apply the Asteroid filterbank frontend to the input. Args: - input (torch.Tensor): Input audio tensor of shape (B, T), where B is - the batch size and T is the number of time steps. - input_length (torch.Tensor): Tensor of shape (B,) containing the - original lengths of each sequence in the batch. + input: Input (B, T). + input_length: Input length (B,). Returns: - Tuple[torch.Tensor, torch.Tensor]: - - Frame-wise output features of shape (B, T', D), where T' is the - number of frames and D is the number of features per frame. - - Updated input lengths after processing, of shape (B,). - - Raises: - AssertionError: If the input tensor does not have exactly 2 dimensions. - - Note: - The forward pass temporarily disables automatic mixed precision to - ensure consistent results. - - Example: - >>> frontend = AsteroidFrontend(sinc_filters=256, sinc_kernel_size=251, sinc_stride=16) - >>> input_tensor = torch.randn(32, 16000) # (batch_size, time) - >>> input_lengths = torch.full((32,), 16000) - >>> output, output_lengths = frontend.forward(input_tensor, input_lengths) - >>> print(output.shape) # (batch_size, time', features) - >>> print(output_lengths.shape) # (batch_size,) + Tensor: Frame-wise output (B, T', D). """ # input check assert ( @@ -150,20 +106,5 @@ def forward( return x.permute(0, 2, 1), input_length def output_size(self) -> int: - """ - Return the output size of the feature dimension. - - This method returns the number of features in the output of the - Asteroid filterbank frontend, which is equal to the number of - sinc filters used in the convolutional layer. - - Returns: - int: The number of features in the output, corresponding to - the number of sinc filters. - - Example: - >>> frontend = AsteroidFrontend(sinc_filters=256) - >>> output_dim = frontend.output_size() - >>> print(output_dim) # 256 - """ + """Return output length of feature dimension D.""" return self.sinc_filters diff --git a/espnet2/asr/frontend/default.py b/espnet2/asr/frontend/default.py index 2c76dba7164..1cceef269d5 100644 --- a/espnet2/asr/frontend/default.py +++ b/espnet2/asr/frontend/default.py @@ -15,49 +15,9 @@ class DefaultFrontend(AbsFrontend): - """ - Conventional frontend structure for ASR. - - This class implements a standard frontend processing pipeline for Automatic Speech Recognition (ASR), - consisting of the following steps: STFT -> WPE -> MVDR-Beamformer -> Power-spectrum -> Log-Mel-Fbank. - - Attributes: - stft (Stft): Short-time Fourier transform module. - frontend (Frontend): Speech enhancement frontend module. - logmel (LogMel): Log-Mel filterbank feature extraction module. - n_mels (int): Number of Mel filterbank channels. - frontend_type (str): Type of frontend, set to "default". - hop_length (int): Hop length for STFT. - apply_stft (bool): Flag to determine whether to apply STFT. - - Args: - fs (Union[int, str]): Sampling frequency of the input audio. Defaults to 16000. - n_fft (int): FFT size. Defaults to 512. - win_length (Optional[int]): Window length for STFT. Defaults to None. - hop_length (int): Hop length for STFT. Defaults to 128. - window (Optional[str]): Window function type. Defaults to "hann". - center (bool): Whether to pad the input on both sides. Defaults to True. - normalized (bool): Whether to normalize the STFT. Defaults to False. - onesided (bool): Whether to return only one-sided spectrum. Defaults to True. - n_mels (int): Number of Mel filterbank channels. Defaults to 80. - fmin (Optional[int]): Minimum frequency for Mel filters. Defaults to None. - fmax (Optional[int]): Maximum frequency for Mel filters. Defaults to None. - htk (bool): Whether to use HTK formula for Mel scale. Defaults to False. - frontend_conf (Optional[dict]): Configuration for the Frontend module. Defaults to None. - apply_stft (bool): Whether to apply STFT. Defaults to True. - - Note: - This class inherits from AbsFrontend and implements the conventional frontend structure - used in many ASR systems. It combines multiple processing steps to convert raw audio - input into features suitable for acoustic modeling. - - Examples: - >>> frontend = DefaultFrontend(fs=16000, n_fft=512, n_mels=80) - >>> input_audio = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> features, feat_lengths = frontend(input_audio, input_lengths) - >>> features.shape - torch.Size([1, 126, 80]) + """Conventional frontend structure for ASR. + + Stft -> WPE -> MVDR-Beamformer -> Power-spec -> Log-Mel-Fbank """ @typechecked @@ -117,64 +77,11 @@ def __init__( self.frontend_type = "default" def output_size(self) -> int: - """ - Returns the output size of the frontend. - - Returns: - int: The number of Mel filterbank channels (n_mels) used in the frontend. - - Note: - This method is used to determine the dimensionality of the feature vectors - produced by the frontend, which is essential for configuring subsequent - components in the ASR pipeline. - - Examples: - >>> frontend = DefaultFrontend(n_mels=80) - >>> frontend.output_size() - 80 - """ return self.n_mels def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Perform frontend processing on the input audio. - - This method applies the complete frontend processing pipeline to the input audio, - including STFT, optional speech enhancement, channel selection, power spectrum - computation, and log-mel feature extraction. - - Args: - input (torch.Tensor): Input audio tensor of shape (Batch, Time). - input_lengths (torch.Tensor): Tensor of input audio lengths of shape (Batch,). - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - input_feats (torch.Tensor): Processed features of shape (Batch, Length, Dim). - - feats_lens (torch.Tensor): Lengths of processed features of shape (Batch,). - - Note: - The processing steps include: - 1. Domain conversion (e.g., STFT) - 2. Optional speech enhancement - 3. Channel selection for multi-channel input - 4. Power spectrum computation - 5. Log-mel feature extraction - - During training, a random channel is selected for multi-channel input. - During inference, the first channel is used. - - Examples: - >>> frontend = DefaultFrontend(fs=16000, n_fft=512, n_mels=80) - >>> input_audio = torch.randn(2, 16000) # 2 utterances of 1 second each - >>> input_lengths = torch.tensor([16000, 12000]) # Second utterance is shorter - >>> features, feat_lengths = frontend(input_audio, input_lengths) - >>> features.shape - torch.Size([2, 126, 80]) - >>> feat_lengths - tensor([126, 95]) - """ # 1. Domain-conversion: e.g. Stft: time -> time-freq if self.stft is not None: input_stft, feats_lens = self._compute_stft(input, input_lengths) diff --git a/espnet2/asr/frontend/fused.py b/espnet2/asr/frontend/fused.py index 6bd5762c9b0..ab4cd7fdbe8 100644 --- a/espnet2/asr/frontend/fused.py +++ b/espnet2/asr/frontend/fused.py @@ -10,41 +10,6 @@ class FusedFrontends(AbsFrontend): - """ - A class that combines multiple frontend modules for feature extraction in speech processing. - - This class allows the fusion of different frontend modules, such as DefaultFrontend and S3prlFrontend, - and aligns their outputs using a specified method (currently only linear projection is supported). - - Attributes: - align_method (str): The method used to align and fuse frontend outputs. - proj_dim (int): The dimension of the projection applied to each frontend's output. - frontends (torch.nn.ModuleList): A list of frontend modules to be combined. - gcd (int): The greatest common divisor of hop lengths across all frontends. - factors (list): A list of factors used for reshaping frontend outputs. - projection_layers (torch.nn.ModuleList): Linear projection layers for each frontend. - - Args: - frontends (list): A list of dictionaries, each containing configuration for a frontend. - align_method (str, optional): The method used to align frontend outputs. Defaults to "linear_projection". - proj_dim (int, optional): The dimension of the projection applied to each frontend's output. Defaults to 100. - fs (int, optional): The sampling frequency of the input audio. Defaults to 16000. - - Raises: - NotImplementedError: If an unsupported frontend type is specified. - - Note: - Currently, only 'default' and 's3prl' frontend types are supported. - The 'linear_projection' is the only supported alignment method at the moment. - - Examples: - >>> frontends = [ - ... {"frontend_type": "default", "n_mels": 80}, - ... {"frontend_type": "s3prl", "frontend_conf": "wav2vec2_base"} - ... ] - >>> fused_frontend = FusedFrontends(frontends=frontends, proj_dim=128) - """ - @typechecked def __init__( self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000 @@ -134,55 +99,11 @@ def __init__( self.projection_layers = self.projection_layers.to(torch.device(dev)) def output_size(self) -> int: - """ - Returns the output size of the fused frontends. - - This method calculates the total output size of all combined frontends - after projection and fusion. - - Returns: - int: The total output size, which is the product of the number of - frontends and the projection dimension (proj_dim). - - Example: - >>> fused_frontend = FusedFrontends(frontends=[...], proj_dim=100) - >>> output_size = fused_frontend.output_size() - >>> print(output_size) - 200 # Assuming two frontends are used - """ return len(self.frontends) * self.proj_dim def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Processes the input through all frontends and fuses their outputs. - - This method applies each frontend to the input, projects the outputs, - aligns them, and concatenates them to produce a single fused feature representation. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, num_samples). - input_lengths (torch.Tensor): Tensor of input lengths for each sample in the batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - input_feats (torch.Tensor): The fused features from all frontends. - Shape: (batch_size, num_frames, total_proj_dim). - - feats_lens (torch.Tensor): The lengths of the feature sequences for each sample in the batch. - - Raises: - NotImplementedError: If an unsupported alignment method is specified. - - Note: - Currently, only the 'linear_projection' alignment method is implemented. - - Example: - >>> fused_frontend = FusedFrontends(frontends=[...]) - >>> input_tensor = torch.randn(32, 16000) # (batch_size, num_samples) - >>> input_lengths = torch.full((32,), 16000) - >>> fused_features, feature_lengths = fused_frontend.forward(input_tensor, input_lengths) - """ # step 0 : get all frontends features self.feats = [] for frontend in self.frontends: diff --git a/espnet2/asr/frontend/melspec_torch.py b/espnet2/asr/frontend/melspec_torch.py index 0e8d33563cb..26a1f108f21 100644 --- a/espnet2/asr/frontend/melspec_torch.py +++ b/espnet2/asr/frontend/melspec_torch.py @@ -15,32 +15,7 @@ class MelSpectrogramTorch(AbsFrontend): - """ - Mel-Spectrogram using Torchaudio Implementation. - - This class implements a Mel-Spectrogram frontend using Torchaudio's MelSpectrogram - transform. It can optionally apply pre-emphasis, logarithmic scaling, and - normalization to the input audio signal. - - Attributes: - log (bool): Whether to apply logarithmic scaling to the spectrogram. - n_mels (int): Number of Mel filterbanks. - preemp (bool): Whether to apply pre-emphasis to the input signal. - normalize (Optional[str]): Normalization method ('mn' for mean normalization or None). - window_fn (Callable): Window function (torch.hann_window or torch.hamming_window). - flipped_filter (torch.Tensor): Pre-emphasis filter coefficients. - transform (torchaudio.transforms.MelSpectrogram): Mel-spectrogram transform object. - - Note: - This class inherits from AbsFrontend and is designed to be used as a frontend - in speech processing tasks, particularly in the ESPnet2 framework. - - Example: - >>> frontend = MelSpectrogramTorch(n_mels=80, n_fft=512, win_length=400, hop_length=160) - >>> input_signal = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> mel_spec, output_lengths = frontend(input_signal, input_lengths) - """ + """Mel-Spectrogram using Torchaudio Implementation.""" @typechecked def __init__( @@ -88,37 +63,6 @@ def __init__( def forward( self, input: torch.Tensor, input_length: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Compute the Mel-spectrogram of the input audio signal. - - This method applies the Mel-spectrogram transformation to the input audio signal, - optionally performing pre-emphasis, logarithmic scaling, and normalization. - - Args: - input (torch.Tensor): Input audio signal tensor of shape (batch_size, num_samples). - input_length (torch.Tensor): Tensor containing the lengths of each audio signal - in the batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - torch.Tensor: Mel-spectrogram features of shape (batch_size, num_frames, n_mels). - - torch.Tensor: Output lengths tensor containing the number of frames for each - spectrogram in the batch. - - Raises: - AssertionError: If the input tensor does not have exactly 2 dimensions. - - Note: - This method uses torch.no_grad() and torch.cuda.amp.autocast(enabled=False) - to ensure consistent behavior and avoid unnecessary gradient computations. - - Example: - >>> frontend = MelSpectrogramTorch(n_mels=80, n_fft=512, win_length=400, hop_length=160) - >>> input_signal = torch.randn(2, 16000) # Batch of 2 audio signals - >>> input_lengths = torch.tensor([16000, 16000]) - >>> mel_spec, output_lengths = frontend(input_signal, input_lengths) - >>> print(mel_spec.shape) # Expected output: torch.Size([2, num_frames, 80]) - """ # input check assert ( len(input.size()) == 2 @@ -154,20 +98,5 @@ def forward( return x.permute(0, 2, 1), input_length def output_size(self) -> int: - """ - Return the output size of the Mel-spectrogram feature dimension. - - This method returns the number of Mel filterbanks used in the Mel-spectrogram - computation, which corresponds to the size of the feature dimension in the - output. - - Returns: - int: The number of Mel filterbanks (n_mels) used in the Mel-spectrogram - computation. - - Example: - >>> frontend = MelSpectrogramTorch(n_mels=80) - >>> feature_dim = frontend.output_size() - >>> print(feature_dim) # Expected output: 80 - """ + """Return output length of feature dimension D.""" return self.n_mels diff --git a/espnet2/asr/frontend/s3prl.py b/espnet2/asr/frontend/s3prl.py index 191957f5ab2..39ccefb56a4 100644 --- a/espnet2/asr/frontend/s3prl.py +++ b/espnet2/asr/frontend/s3prl.py @@ -12,40 +12,7 @@ class S3prlFrontend(AbsFrontend): - """ - Speech Pretrained Representation frontend structure for ASR. - - This class implements a frontend for Automatic Speech Recognition (ASR) using - pretrained speech representations from the S3PRL toolkit. It supports various - upstream models and allows for flexible configuration of the frontend. - - Attributes: - frontend_type (str): Type of the frontend, set to "s3prl". - hop_length (int): The hop length (downsampling rate) of the featurizer. - tile_factor (int): Factor by which to tile the output representations. - multilayer_feature (bool): Whether to use multilayer features or not. - layer (int): Specific layer to use from the upstream model (-1 for last layer). - - Args: - fs (Union[int, str]): Sampling frequency of the input audio. Defaults to 16000. - frontend_conf (Optional[dict]): Configuration for the frontend. Defaults to None. - download_dir (Optional[str]): Directory to download S3PRL models. Defaults to None. - multilayer_feature (bool): Whether to use multilayer features. Defaults to False. - layer (int): Specific layer to use from the upstream model. Defaults to -1. - - Raises: - Exception: If S3PRL is not properly installed. - - Note: - - All upstream models in S3PRL currently only support 16 kHz audio. - - When a specific layer is selected, multilayer_feature will be deactivated. - - Examples: - >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) - >>> input_tensor = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> feats, feats_lens = frontend(input_tensor, input_lengths) - """ + """Speech Pretrained Representation frontend structure for ASR.""" @typechecked def __init__( @@ -124,53 +91,11 @@ def _tile_representations(self, feature): return tiled_feature def output_size(self) -> int: - """ - Returns the output size of the frontend. - - This method provides the dimensionality of the feature vectors produced by the frontend. - - Returns: - int: The size of the output feature vectors. - - Example: - >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) - >>> output_dim = frontend.output_size() - >>> print(output_dim) - 768 # Example output, actual value may vary depending on the upstream model - """ return self.featurizer.output_size def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Processes the input audio and returns the extracted features. - - This method takes the input audio tensor and its corresponding lengths, passes it through - the S3PRL upstream model and featurizer, and returns the extracted features. - - Args: - input (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). - input_lengths (torch.Tensor): Lengths of each audio in the batch of shape (batch_size,). - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - feats (torch.Tensor): Extracted features of shape (batch_size, num_frames, feature_dim). - - feats_lens (torch.Tensor): Lengths of each feature sequence in the batch of shape (batch_size,). - - Note: - - If a specific layer is selected (self.layer != -1), only features from that layer are returned. - - If multilayer_feature is True, features from multiple layers are combined. - - If tile_factor is not 1, the output features are tiled accordingly. - - Example: - >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) - >>> input_tensor = torch.randn(2, 16000) # 2 audio samples of 1 second each - >>> input_lengths = torch.tensor([16000, 16000]) - >>> feats, feats_lens = frontend(input_tensor, input_lengths) - >>> print(feats.shape) - torch.Size([2, 50, 768]) # Example output, actual shape may vary - """ feats, feats_lens = self.upstream(input, input_lengths) if self.layer != -1: layer = self.layer @@ -188,20 +113,5 @@ def forward( return feats, feats_lens def reload_pretrained_parameters(self): - """ - Reloads the pretrained parameters of the S3PRL frontend model. - - This method restores the original pretrained parameters of the upstream S3PRL model, - effectively resetting any fine-tuning or modifications made to the model weights. - - Note: - This method is useful when you want to reset the model to its initial pretrained state, - for example, after fine-tuning or experimenting with the model parameters. - - Example: - >>> frontend = S3prlFrontend(fs=16000, frontend_conf={'upstream': 'wav2vec2'}) - >>> # After some operations or fine-tuning - >>> frontend.reload_pretrained_parameters() - >>> print("Pretrained S3PRL frontend model parameters reloaded!") - """ + self.upstream.load_state_dict(self.pretrained_params) logging.info("Pretrained S3PRL frontend model parameters reloaded!") diff --git a/espnet2/asr/frontend/whisper.py b/espnet2/asr/frontend/whisper.py index 35ec4b4fdea..b879d606a42 100644 --- a/espnet2/asr/frontend/whisper.py +++ b/espnet2/asr/frontend/whisper.py @@ -10,41 +10,9 @@ class WhisperFrontend(AbsFrontend): - """ - Speech representation frontend using OpenAI's Whisper model encoder outputs. - - This class implements a frontend for speech representation using the encoder - outputs from OpenAI's Whisper model. It processes input audio to generate - log-mel spectrograms and then encodes them using the Whisper encoder. - - Attributes: - n_fft (int): Size of the FFT for spectrogram computation. - win_length (int): Window length for spectrogram computation. - hop_length (int): Hop length for spectrogram computation. - n_mels (int): Number of mel filterbanks. - mel_filters (function): Function to generate mel filterbanks. - pad_or_trim (function): Function to pad or trim input sequences. - whisper (whisper.Whisper): Loaded Whisper model. - freeze_weights (bool): Whether to freeze Whisper model weights. - - Args: - whisper_model (str): Name of the Whisper model to use. Defaults to "small". - fs (Union[int, str]): Sampling frequency in Hz. Defaults to 16000. - freeze_weights (bool): Whether to freeze Whisper model weights. Defaults to True. - download_dir (Optional[str]): Directory to download Whisper model. Defaults to None. - - Raises: - Exception: If the whisper package is not properly installed. - - Note: - This class requires the whisper package to be installed. - The Whisper model only supports 16 kHz audio. + """Speech Representation Using Encoder Outputs from OpenAI's Whisper Model: - Examples: - >>> frontend = WhisperFrontend("small", fs=16000) - >>> input_tensor = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> output, output_lengths = frontend(input_tensor, input_lengths) + URL: https://github.com/openai/whisper """ @typechecked @@ -90,21 +58,6 @@ def __init__( self.freeze_weights = freeze_weights def output_size(self) -> int: - """ - Returns the output size of the Whisper frontend. - - This method returns the dimensionality of the feature vectors produced by - the Whisper encoder. - - Returns: - int: The size of the output feature vectors. - - Example: - >>> frontend = WhisperFrontend("small") - >>> output_dim = frontend.output_size() - >>> print(output_dim) - 512 # This may vary depending on the Whisper model used - """ return self.whisper.encoder.ln_post.normalized_shape[-1] def log_mel_spectrogram( @@ -112,34 +65,6 @@ def log_mel_spectrogram( audio: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: - """ - Compute log-mel spectrogram features from input audio. - - This method computes the log-mel spectrogram features from the input audio - using the Whisper model's preprocessing steps. - - Args: - audio (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). - ilens (torch.Tensor, optional): Tensor of input lengths for each audio in the batch. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - log_spec (torch.Tensor): Log-mel spectrogram features of shape - (batch_size, n_mels, num_frames). - - olens (Optional[torch.Tensor]): Tensor of output lengths for each - spectrogram in the batch. None if ilens is None. - - Note: - The method applies normalization to the log-mel spectrogram as per - Whisper's preprocessing. - - Example: - >>> frontend = WhisperFrontend("small") - >>> audio = torch.randn(1, 16000) - >>> log_spec, olens = frontend.log_mel_spectrogram(audio) - >>> print(log_spec.shape) - torch.Size([1, 80, 100]) # Exact shape may vary based on input length - """ window = torch.hann_window(self.win_length).to(audio.device) stft = torch.stft( audio, self.n_fft, self.hop_length, window=window, return_complex=True @@ -171,34 +96,6 @@ def whisper_encode( input: torch.Tensor, ilens: torch.Tensor = None, ) -> torch.Tensor: - """ - Encode input features using the Whisper encoder. - - This method processes the input features (typically log-mel spectrograms) - through the Whisper encoder to produce high-level representations. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, n_mels, num_frames). - ilens (torch.Tensor, optional): Tensor of input lengths for each feature in the batch. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - x (torch.Tensor): Encoded features of shape (batch_size, num_frames, encoder_dim). - - olens (Optional[torch.Tensor]): Tensor of output lengths for each encoded - sequence in the batch. None if ilens is None. - - Note: - The method applies positional embeddings and processes the input through - the Whisper encoder blocks. The output is truncated if it exceeds the - maximum position embedding size. - - Example: - >>> frontend = WhisperFrontend("small") - >>> log_spec = torch.randn(1, 80, 100) - >>> encoded, olens = frontend.whisper_encode(log_spec) - >>> print(encoded.shape) - torch.Size([1, 100, 512]) # Exact shape may vary based on the Whisper model - """ whisper_encoder = self.whisper.encoder x = F.gelu(whisper_encoder.conv1(input)) @@ -236,33 +133,6 @@ def whisper_encode( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Process input audio through the Whisper frontend. - - This method takes raw audio input, computes log-mel spectrograms, and then - encodes them using the Whisper encoder to produce high-level speech representations. - - Args: - input (torch.Tensor): Input audio tensor of shape (batch_size, num_samples). - input_lengths (torch.Tensor): Tensor of input lengths for each audio in the batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - feats (torch.Tensor): Encoded features of shape (batch_size, num_frames, encoder_dim). - - feats_lens (torch.Tensor): Tensor of output lengths for each encoded sequence in the batch. - - Note: - If self.freeze_weights is True, the Whisper encoding step is performed - with torch.no_grad() to prevent gradient computation and weight updates. - - Example: - >>> frontend = WhisperFrontend("small") - >>> audio = torch.randn(1, 16000) - >>> input_lengths = torch.tensor([16000]) - >>> feats, feats_lens = frontend(audio, input_lengths) - >>> print(feats.shape) - torch.Size([1, 100, 512]) # Exact shape may vary based on input and Whisper model - """ feats, feats_lens = self.log_mel_spectrogram(input, input_lengths) with torch.no_grad() if self.freeze_weights else contextlib.nullcontext(): diff --git a/espnet2/asr/frontend/windowing.py b/espnet2/asr/frontend/windowing.py index 751148b8bf5..f4a34d68e94 100644 --- a/espnet2/asr/frontend/windowing.py +++ b/espnet2/asr/frontend/windowing.py @@ -13,35 +13,17 @@ class SlidingWindow(AbsFrontend): - """ - Sliding Window for raw audio input data. - - This class implements a sliding window operation over batched continuous raw audio tensors. - It is designed to be used as a frontend in audio processing pipelines, particularly - in combination with pre-encoders compatible with raw audio data (e.g., Sinc convolutions). - - The sliding window operation segments the input audio into overlapping frames, - which can be further processed by subsequent layers in the neural network. - - Attributes: - win_length (int): Length of each window frame. - hop_length (int): Number of samples to advance between successive frames. - channels (int): Number of input audio channels. - padding (Optional[int]): Padding option (currently not implemented). - fs (Optional[int]): Sampling rate (placeholder for compatibility, not used). + """Sliding Window. - Note: - - Output length is calculated incorrectly if audio is shorter than win_length. - - Trailing values are currently discarded as padding is not yet implemented. - - No additional window function is applied to input values. + Provides a sliding window over a batched continuous raw audio tensor. + Optionally, provides padding (Currently not implemented). + Combine this module with a pre-encoder compatible with raw audio data, + for example Sinc convolutions. - Examples: - >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) - >>> input_tensor = torch.randn(32, 16000, 1) # (batch_size, time_steps, channels) - >>> input_lengths = torch.full((32,), 16000) - >>> output, output_lengths = sliding_window(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 98, 1, 400]) + Known issues: + Output length is calculated incorrectly if audio shorter than win_length. + WARNING: trailing values are discarded - padding not implemented yet. + There is currently no additional window function applied to input values. """ @typechecked @@ -72,38 +54,15 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Apply a sliding window on the input. - - This method processes the input audio tensor by applying a sliding window, - segmenting the continuous audio into overlapping frames. + """Apply a sliding window on the input. Args: - input (torch.Tensor): Input tensor of shape (B, T, C*D) or (B, T*C*D), - where B is batch size, T is time steps, C is channels, and D is 1. - input_lengths (torch.Tensor): Tensor containing the valid length of each - sample in the batch. + input: Input (B, T, C*D) or (B, T*C*D), with D=C=1. + input_lengths: Input lengths within batch. Returns: - Tuple[torch.Tensor, torch.Tensor]: - - output (torch.Tensor): Windowed output of shape (B, T, C, D), - where D is the window length (win_length). - - output_lengths (torch.Tensor): Tensor containing the valid length - of each windowed sample in the batch. - - Examples: - >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) - >>> input_tensor = torch.randn(32, 16000, 1) # (batch_size, time_steps, channels) - >>> input_lengths = torch.full((32,), 16000) - >>> output, output_lengths = sliding_window.forward(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 98, 1, 400]) - >>> print(output_lengths) - tensor([98, 98, 98, ..., 98, 98, 98]) - - Note: - The output length is calculated as: - (input_length - win_length) // hop_length + 1 + Tensor: Output with dimensions (B, T, C, D), with D=win_length. + Tensor: Output lengths within batch. """ input_size = input.size() B = input_size[0] @@ -125,22 +84,5 @@ def forward( return output, output_lengths def output_size(self) -> int: - """ - Return the output size of the feature dimension. - - This method returns the length of the window, which corresponds to the - size of the feature dimension (D) in the output tensor. - - Returns: - int: The window length (win_length). - - Examples: - >>> sliding_window = SlidingWindow(win_length=400, hop_length=160, channels=1) - >>> print(sliding_window.output_size()) - 400 - - Note: - This value is used to determine the size of the last dimension in the - output tensor returned by the forward method. - """ + """Return output length of feature dimension D, i.e. the window length.""" return self.win_length diff --git a/espnet2/asr/layers/cgmlp.py b/espnet2/asr/layers/cgmlp.py index 3551699c12d..c404bc3828a 100644 --- a/espnet2/asr/layers/cgmlp.py +++ b/espnet2/asr/layers/cgmlp.py @@ -13,39 +13,7 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module): - """ - Convolutional Spatial Gating Unit (CSGU) for MLP with convolutional gating. - - This class implements the Convolutional Spatial Gating Unit, a key component - of the cgMLP architecture. It applies layer normalization, 1D convolution, - and optional linear transformation to the input, followed by an activation - function and dropout. - - Attributes: - norm (LayerNorm): Layer normalization applied to the input. - conv (torch.nn.Conv1d): 1D convolution layer. - linear (torch.nn.Linear or None): Optional linear layer after convolution. - act (torch.nn.Module): Activation function for gating. - dropout (torch.nn.Dropout): Dropout layer. - - Args: - size (int): Input size (should be divisible by 2). - kernel_size (int): Kernel size for the 1D convolution. - dropout_rate (float): Dropout rate. - use_linear_after_conv (bool): Whether to use a linear layer after convolution. - gate_activation (str): Activation function to use for gating. - - Note: - The input channels are split in half, with one half used for gating - the other half. - - Example: - >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') - >>> input_tensor = torch.randn(32, 100, 512) - >>> output = csgu(input_tensor) - >>> output.shape - torch.Size([32, 100, 256]) - """ + """Convolutional Spatial Gating Unit (CSGU).""" def __init__( self, @@ -80,66 +48,21 @@ def __init__( self.dropout = torch.nn.Dropout(dropout_rate) def espnet_initialization_fn(self): - """ - Initialize the weights of the convolutional and linear layers. - - This method initializes the weights of the convolutional layer and, if present, - the linear layer of the Convolutional Spatial Gating Unit (CSGU). The - initialization follows the strategy described in the original cgMLP paper. - - The convolutional layer's weights are initialized from a normal distribution - with a small standard deviation, while its biases are initialized to ones. - If a linear layer is present, its weights are similarly initialized from a - normal distribution, and its biases are set to ones. - - Returns: - None - - Note: - This method is typically called during the initialization of the CSGU - module and doesn't need to be called manually in most cases. - - Example: - >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') - >>> csgu.espnet_initialization_fn() - """ + torch.nn.init.normal_(self.conv.weight, std=1e-6) torch.nn.init.ones_(self.conv.bias) if self.linear is not None: torch.nn.init.normal_(self.linear.weight, std=1e-6) torch.nn.init.ones_(self.linear.bias) def forward(self, x, gate_add=None): - """ - Forward pass of the Convolutional Spatial Gating Unit. - - This method performs the forward computation of the CSGU. It splits the input - tensor into two halves along the last dimension, applies normalization and - convolution to one half, and uses it to gate the other half. + """Forward method Args: - x (torch.Tensor): Input tensor of shape (N, T, D), where N is the batch size, - T is the sequence length, and D is the feature dimension. - gate_add (torch.Tensor, optional): Additional tensor to be added to the gate - values. Should have shape (N, T, D/2). Defaults to None. + x (torch.Tensor): (N, T, D) + gate_add (torch.Tensor): (N, T, D/2) Returns: - torch.Tensor: Output tensor of shape (N, T, D/2). The output dimension is - half of the input dimension due to the gating mechanism. - - Raises: - ValueError: If the input tensor's last dimension is not even. - - Example: - >>> csgu = ConvolutionalSpatialGatingUnit(512, 3, 0.1, True, 'gelu') - >>> input_tensor = torch.randn(32, 100, 512) - >>> output = csgu(input_tensor) - >>> output.shape - torch.Size([32, 100, 256]) - - Note: - The input tensor is split into two halves: x_r and x_g. x_g is processed - through normalization, convolution, and optional linear transformation, - then used to gate x_r. The result is then passed through dropout. + out (torch.Tensor): (N, T, D/2) """ x_r, x_g = x.chunk(2, dim=-1) @@ -159,42 +82,7 @@ def forward(self, x, gate_add=None): class ConvolutionalGatingMLP(torch.nn.Module): - """ - Convolutional Gating MLP (cgMLP) module. - - This class implements the Convolutional Gating MLP, a variant of the MLP - architecture that incorporates convolutional gating mechanisms. It consists - of two channel projection layers and a Convolutional Spatial Gating Unit (CSGU). - - The cgMLP processes input by first projecting it to a higher dimension, - then applying the CSGU for feature interaction and gating, and finally - projecting it back to the original dimension. - - Attributes: - channel_proj1 (torch.nn.Sequential): First channel projection layer with GELU activation. - csgu (ConvolutionalSpatialGatingUnit): The core CSGU component. - channel_proj2 (torch.nn.Linear): Second channel projection layer. - - Args: - size (int): Input and output size of the module. - linear_units (int): Number of units in the intermediate linear layer. - kernel_size (int): Kernel size for the convolutional layer in CSGU. - dropout_rate (float): Dropout rate used in CSGU. - use_linear_after_conv (bool): Whether to use a linear layer after convolution in CSGU. - gate_activation (str): Activation function to use for gating in CSGU. - - Example: - >>> cgmlp = ConvolutionalGatingMLP(512, 1024, 3, 0.1, True, 'gelu') - >>> input_tensor = torch.randn(32, 100, 512) - >>> output = cgmlp(input_tensor, None) - >>> output.shape - torch.Size([32, 100, 512]) - - Note: - The forward method of this class can handle inputs with or without - positional embeddings. If positional embeddings are provided, they - will be returned along with the processed tensor. - """ + """Convolutional Gating MLP (cgMLP).""" def __init__( self, @@ -220,44 +108,6 @@ def __init__( self.channel_proj2 = torch.nn.Linear(linear_units // 2, size) def forward(self, x, mask): - """ - Forward pass of the Convolutional Gating MLP. - - This method performs the forward computation of the cgMLP. It processes - the input through two channel projection layers and a Convolutional - Spatial Gating Unit (CSGU). - - Args: - x (torch.Tensor or tuple): If tuple, it should contain: - - xs_pad (torch.Tensor): Input tensor of shape (N, T, D), where N is - the batch size, T is the sequence length, and D is the feature dimension. - - pos_emb (torch.Tensor): Positional embedding tensor. - If not a tuple, it's treated as the input tensor itself. - mask (torch.Tensor): Mask tensor (not used in the current implementation). - - Returns: - torch.Tensor or tuple: If the input included positional embeddings, returns a tuple - containing: - - xs_pad (torch.Tensor): Processed tensor of shape (N, T, D). - - pos_emb (torch.Tensor): The input positional embedding tensor. - Otherwise, returns just the processed tensor. - - Example: - >>> cgmlp = ConvolutionalGatingMLP(512, 1024, 3, 0.1, True, 'gelu') - >>> input_tensor = torch.randn(32, 100, 512) - >>> output = cgmlp(input_tensor, None) - >>> output.shape - torch.Size([32, 100, 512]) - - >>> input_with_pos = (input_tensor, torch.randn(32, 100, 512)) - >>> output_with_pos = cgmlp(input_with_pos, None) - >>> isinstance(output_with_pos, tuple) - True - - Note: - The 'mask' argument is currently not used in the method but is kept - for compatibility with other modules in the architecture. - """ if isinstance(x, tuple): xs_pad, pos_emb = x else: diff --git a/espnet2/asr/layers/fastformer.py b/espnet2/asr/layers/fastformer.py index 6cfefb476a9..24ca9475c15 100644 --- a/espnet2/asr/layers/fastformer.py +++ b/espnet2/asr/layers/fastformer.py @@ -12,44 +12,7 @@ class FastSelfAttention(torch.nn.Module): - """ - Fast self-attention mechanism used in Fastformer. - - This class implements the fast self-attention mechanism as described in the - Fastformer paper by Wu et al. It provides an efficient alternative to - traditional self-attention mechanisms used in transformer models. - - Attributes: - attention_head_size (int): The size of each attention head. - num_attention_heads (int): The number of attention heads. - query (torch.nn.Linear): Linear layer for query transformation. - query_att (torch.nn.Linear): Linear layer for query attention. - key (torch.nn.Linear): Linear layer for key transformation. - key_att (torch.nn.Linear): Linear layer for key attention. - transform (torch.nn.Linear): Linear layer for final transformation. - dropout (torch.nn.Dropout): Dropout layer for regularization. - - Args: - size (int): The input size (hidden size) of the model. - attention_heads (int): The number of attention heads. - dropout_rate (float): The dropout rate to use for regularization. - - Raises: - ValueError: If the hidden size is not an integer multiple of the number - of attention heads. - - Example: - >>> model = FastSelfAttention(512, 8, 0.1) - >>> input_tensor = torch.randn(32, 100, 512) # (batch, seq_len, hidden_size) - >>> mask = torch.ones(32, 1, 100) # (batch, 1, seq_len) - >>> output = model(input_tensor, mask) - >>> print(output.shape) - torch.Size([32, 100, 512]) - - Note: - This implementation is based on the paper "Fastformer: Additive Attention - Can Be All You Need" by Wu et al. and the corresponding GitHub repository. - """ + """Fast self-attention used in Fastformer.""" def __init__( self, @@ -74,72 +37,22 @@ def __init__( self.dropout = torch.nn.Dropout(dropout_rate) def espnet_initialization_fn(self): - """ - Initialize the weights of the FastSelfAttention module. - - This method applies the `init_weights` function to all submodules of the - FastSelfAttention module. It is designed to be used as an initialization - function in the ESPnet framework. - - Note: - This method does not take any arguments and does not return anything. - It modifies the module's weights in-place. - - Example: - >>> model = FastSelfAttention(512, 8, 0.1) - >>> model.espnet_initialization_fn() - """ + self.apply(self.init_weights) def init_weights(self, module): - """ - Initialize the weights of a given module. - - This method initializes the weights of linear layers in the module. It sets - the weights to a normal distribution with mean 0.0 and standard deviation 0.02, - and initializes biases to zero. - - Args: - module (torch.nn.Module): The module whose weights are to be initialized. - - Note: - This method is typically called by `espnet_initialization_fn` and is not - meant to be used directly in most cases. - - Example: - >>> linear = torch.nn.Linear(10, 10) - >>> model = FastSelfAttention(512, 8, 0.1) - >>> model.init_weights(linear) - """ if isinstance(module, torch.nn.Linear): module.weight.data.normal_(mean=0.0, std=0.02) if isinstance(module, torch.nn.Linear) and module.bias is not None: module.bias.data.zero_() def transpose_for_scores(self, x): - """ - Reshape and transpose the input tensor for computing attention scores. - - This method reshapes the input tensor and transposes it to prepare for - computing attention scores in the fast self-attention mechanism. + """Reshape and transpose to compute scores. Args: - x (torch.Tensor): Input tensor of shape (batch, time, size), - where size = n_heads * attn_dim. + x: (batch, time, size = n_heads * attn_dim) Returns: - torch.Tensor: Reshaped and transposed tensor of shape - (batch, n_heads, time, attn_dim). - - Example: - >>> model = FastSelfAttention(512, 8, 0.1) - >>> x = torch.randn(32, 100, 512) - >>> transposed = model.transpose_for_scores(x) - >>> print(transposed.shape) - torch.Size([32, 8, 100, 64]) - - Note: - This method is used internally in the forward pass of the FastSelfAttention - module and is not typically called directly by users. + (batch, n_heads, time, attn_dim) """ new_x_shape = x.shape[:-1] + ( @@ -149,34 +62,14 @@ def transpose_for_scores(self, x): return x.reshape(*new_x_shape).transpose(1, 2) def forward(self, xs_pad, mask): - """ - Perform the forward pass of the FastSelfAttention module. - - This method implements the fast self-attention mechanism, processing the input - tensor and applying the attention weights to produce the output. + """Forward method. Args: - xs_pad (torch.Tensor): Input tensor of shape (batch, time, size), - where size = n_heads * attn_dim. - mask (torch.Tensor): Mask tensor of shape (batch, 1, time), where - non-padding positions are 1 and padding positions are 0. + xs_pad: (batch, time, size = n_heads * attn_dim) + mask: (batch, 1, time), nonpadding is 1, padding is 0 Returns: - torch.Tensor: Output tensor after applying fast self-attention, - of shape (batch, time, size). - - Example: - >>> model = FastSelfAttention(512, 8, 0.1) - >>> xs_pad = torch.randn(32, 100, 512) - >>> mask = torch.ones(32, 1, 100) - >>> output = model.forward(xs_pad, mask) - >>> print(output.shape) - torch.Size([32, 100, 512]) - - Note: - The mask tensor should have 1s for non-padding positions and 0s for - padding positions. The method internally inverts this mask for - compatibility with the implementation. + torch.Tensor: (batch, time, size) """ batch_size, seq_len, _ = xs_pad.shape diff --git a/espnet2/asr/maskctc_model.py b/espnet2/asr/maskctc_model.py index 499767c30d5..b3960359340 100644 --- a/espnet2/asr/maskctc_model.py +++ b/espnet2/asr/maskctc_model.py @@ -37,53 +37,7 @@ def autocast(enabled=True): class MaskCTCModel(ESPnetASRModel): - """ - Hybrid CTC/Masked LM Encoder-Decoder model (Mask-CTC). - - This class implements the Mask-CTC model, which combines Connectionist Temporal - Classification (CTC) and Masked Language Model (MLM) approaches for automatic - speech recognition (ASR). - - The model consists of an encoder, a CTC branch, and an MLM decoder branch. It - supports various components such as frontend processing, spectrogram - augmentation, normalization, pre-encoding, post-encoding, and joint network - functionalities. - - Attributes: - vocab_size (int): Size of the vocabulary, including the mask token. - token_list (List[str]): List of tokens in the vocabulary. - mask_token (int): Token ID for the mask token. - criterion_mlm (LabelSmoothingLoss): Loss function for the MLM branch. - error_calculator (ErrorCalculator): Calculator for CER and WER metrics. - - Args: - vocab_size (int): Size of the vocabulary. - token_list (Union[Tuple[str, ...], List[str]]): List of tokens in the vocabulary. - frontend (Optional[AbsFrontend]): Frontend processing module. - specaug (Optional[AbsSpecAug]): Spectrogram augmentation module. - normalize (Optional[AbsNormalize]): Normalization module. - preencoder (Optional[AbsPreEncoder]): Pre-encoder module. - encoder (AbsEncoder): Encoder module. - postencoder (Optional[AbsPostEncoder]): Post-encoder module. - decoder (MLMDecoder): Masked Language Model decoder. - ctc (CTC): Connectionist Temporal Classification module. - joint_network (Optional[torch.nn.Module]): Joint network module. - ctc_weight (float): Weight for the CTC loss (default: 0.5). - interctc_weight (float): Weight for intermediate CTC loss (default: 0.0). - ignore_id (int): ID to be ignored in loss computation (default: -1). - lsm_weight (float): Label smoothing weight (default: 0.0). - length_normalized_loss (bool): Whether to normalize loss by length (default: False). - report_cer (bool): Whether to report Character Error Rate (default: True). - report_wer (bool): Whether to report Word Error Rate (default: True). - sym_space (str): Space symbol (default: ""). - sym_blank (str): Blank symbol (default: ""). - sym_mask (str): Mask symbol (default: ""). - extract_feats_in_collect_stats (bool): Whether to extract features in collect_stats (default: True). - - Note: - This model extends the ESPnetASRModel and modifies it to incorporate - the Mask-CTC approach, which combines CTC and MLM for improved ASR performance. - """ + """Hybrid CTC/Masked LM Encoder-Decoder model (Mask-CTC)""" @typechecked def __init__( @@ -166,41 +120,13 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """ - Forward pass for the Mask-CTC model. - - This method performs the forward pass through the entire model, including - frontend processing, encoding, CTC loss calculation, and MLM loss calculation. + """Frontend + Encoder + Decoder + Calc loss Args: - speech (torch.Tensor): Input speech tensor of shape (Batch, Length, ...). - speech_lengths (torch.Tensor): Tensor of input speech lengths of shape (Batch,). - text (torch.Tensor): Target text tensor of shape (Batch, Length). - text_lengths (torch.Tensor): Tensor of target text lengths of shape (Batch,). - **kwargs: Additional keyword arguments. - - Returns: - Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: A tuple containing: - - loss (torch.Tensor): The total loss for the forward pass. - - stats (Dict[str, torch.Tensor]): A dictionary of statistics including - CTC loss, MLM loss, accuracies, and error rates. - - weight (torch.Tensor): The batch size as a weight for the loss. - - Raises: - AssertionError: If the input tensors have inconsistent batch sizes or - if text_lengths is not a 1D tensor. - - Note: - This method calculates both CTC and MLM losses, combining them based on - the specified CTC weight. It also handles intermediate CTC loss if enabled. - - Examples: - >>> speech = torch.randn(2, 1000, 80) - >>> speech_lengths = torch.tensor([1000, 800]) - >>> text = torch.randint(0, 100, (2, 20)) - >>> text_lengths = torch.tensor([20, 15]) - >>> model = MaskCTCModel(...) - >>> loss, stats, weight = model.forward(speech, speech_lengths, text, text_lengths) + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified @@ -320,28 +246,6 @@ def nll( ys_pad: torch.Tensor, ys_pad_lens: torch.Tensor, ) -> torch.Tensor: - """ - Calculate negative log-likelihood for the Mask-CTC model. - - This method is not implemented for the Mask-CTC model and will raise a - NotImplementedError if called. - - Args: - encoder_out (torch.Tensor): Output from the encoder. - encoder_out_lens (torch.Tensor): Lengths of encoder outputs. - ys_pad (torch.Tensor): Padded target sequences. - ys_pad_lens (torch.Tensor): Lengths of target sequences. - - Returns: - torch.Tensor: This method does not return a value. - - Raises: - NotImplementedError: This method is not implemented for the Mask-CTC model. - - Note: - The negative log-likelihood calculation is not applicable to the Mask-CTC - model due to its hybrid nature combining CTC and MLM approaches. - """ raise NotImplementedError def batchify_nll( @@ -352,63 +256,11 @@ def batchify_nll( ys_pad_lens: torch.Tensor, batch_size: int = 100, ): - """ - Batchify the calculation of negative log-likelihood for the Mask-CTC model. - - This method is not implemented for the Mask-CTC model and will raise a - NotImplementedError if called. - - Args: - encoder_out (torch.Tensor): Output from the encoder. - encoder_out_lens (torch.Tensor): Lengths of encoder outputs. - ys_pad (torch.Tensor): Padded target sequences. - ys_pad_lens (torch.Tensor): Lengths of target sequences. - batch_size (int): Size of each batch for processing (default: 100). - - Returns: - This method does not return a value. - - Raises: - NotImplementedError: This method is not implemented for the Mask-CTC model. - - Note: - The batchified negative log-likelihood calculation is not applicable to the - Mask-CTC model due to its hybrid nature combining CTC and MLM approaches. - This method is included for compatibility with other ASR model interfaces - but is not functional in the Mask-CTC context. - """ raise NotImplementedError class MaskCTCInference(torch.nn.Module): - """ - Mask-CTC-based non-autoregressive inference for automatic speech recognition. - - This class implements the inference process for the Mask-CTC model, which combines - Connectionist Temporal Classification (CTC) and Masked Language Model (MLM) approaches - for non-autoregressive speech recognition. - - The inference process involves iterative decoding, where masked tokens are - progressively predicted based on CTC probabilities and MLM predictions. - - Attributes: - ctc (CTC): The CTC module from the ASR model. - mlm (MLMDecoder): The MLM decoder from the ASR model. - mask_token (int): The token ID representing the mask. - n_iterations (int): Number of iterations for the decoding process. - threshold_probability (float): Probability threshold for masking tokens. - converter (TokenIDConverter): Converter for token IDs to text. - - Args: - asr_model (MaskCTCModel): The trained Mask-CTC ASR model. - n_iterations (int): Number of iterations for the decoding process. - threshold_probability (float): Probability threshold for masking tokens. - - Note: - This class is designed to work with the MaskCTCModel and provides a - non-autoregressive inference method that can potentially be faster than - traditional autoregressive decoding approaches. - """ + """Mask-CTC-based non-autoregressive inference""" def __init__( self, @@ -426,62 +278,11 @@ def __init__( self.converter = TokenIDConverter(token_list=asr_model.token_list) def ids2text(self, ids: List[int]): - """ - Convert a list of token IDs to readable text. - - This method converts a sequence of token IDs to a human-readable string, - replacing special tokens with their corresponding symbols. - - Args: - ids (List[int]): A list of token IDs to be converted to text. - - Returns: - str: The converted text string. - - Note: - - The method replaces "" with "_" and "" with a space character. - - This conversion is useful for visualizing the output during the inference process. - - Example: - >>> inference = MaskCTCInference(...) - >>> ids = [1, 2, 3, 4, 5] # Assuming these are valid token IDs - >>> text = inference.ids2text(ids) - >>> print(text) - 'converted text' - """ text = "".join(self.converter.ids2tokens(ids)) return text.replace("", "_").replace("", " ") def forward(self, enc_out: torch.Tensor) -> List[Hypothesis]: - """ - Perform Mask-CTC inference on the given encoder output. - - This method implements the non-autoregressive Mask-CTC inference algorithm, - which iteratively refines the output by predicting masked tokens. - - Args: - enc_out (torch.Tensor): The encoder output tensor of shape (T, D), - where T is the sequence length and D is the feature dimension. - - Returns: - List[Hypothesis]: A list containing a single Hypothesis object with the - predicted token sequence. - - Note: - The inference process involves the following steps: - 1. Generate initial CTC greedy outputs - 2. Mask low-confidence tokens based on CTC probabilities - 3. Iteratively predict masked tokens using the MLM decoder - 4. Finalize the output by predicting any remaining masked tokens - - The method logs intermediate results for debugging purposes. - - Example: - >>> inference = MaskCTCInference(...) - >>> encoder_output = torch.randn(100, 256) # (T, D) - >>> hypothesis = inference(encoder_output) - >>> predicted_tokens = hypothesis[0].yseq - """ + """Perform Mask-CTC inference""" # greedy ctc outputs enc_out = enc_out.unsqueeze(0) ctc_probs, ctc_ids = torch.exp(self.ctc.log_softmax(enc_out)).max(dim=-1) diff --git a/espnet2/asr/pit_espnet_model.py b/espnet2/asr/pit_espnet_model.py index 8e1734e81c8..aa62abbc471 100644 --- a/espnet2/asr/pit_espnet_model.py +++ b/espnet2/asr/pit_espnet_model.py @@ -29,35 +29,6 @@ def autocast(enabled=True): class PITLossWrapper(AbsLossWrapper): - """ - Permutation Invariant Training (PIT) Loss Wrapper. - - This class implements a wrapper for applying Permutation Invariant Training - to a given loss criterion function. It is designed to handle multi-speaker - scenarios where the order of speakers in the inference and reference may not - be consistent. - - Attributes: - criterion_fn (Callable): The loss criterion function to be wrapped. - num_ref (int): The number of reference speakers. - - Args: - criterion_fn (Callable): The loss criterion function to be wrapped. - num_ref (int): The number of reference speakers. - - Note: - This wrapper is similar to the PIT solver in espnet2/enh/loss/wrapper/pit_solver.py. - - Examples: - >>> criterion = torch.nn.MSELoss() - >>> pit_wrapper = PITLossWrapper(criterion, num_ref=2) - >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) - >>> ref = torch.randn(4, 2, 10) # (batch, num_speakers, features) - >>> inf_lens = torch.full((4, 2), 10) - >>> ref_lens = torch.full((4, 2), 10) - >>> loss, perm = pit_wrapper(inf, inf_lens, ref, ref_lens) - """ - def __init__(self, criterion_fn: Callable, num_ref: int): super().__init__() self.criterion_fn = criterion_fn @@ -71,44 +42,15 @@ def forward( ref_lens: torch.Tensor, others: Dict = None, ): - """ - Apply Permutation Invariant Training (PIT) loss calculation. - - This method computes the PIT loss by finding the optimal permutation of - speakers that minimizes the total loss across all possible permutations. + """PITLoss Wrapper function. Similar to espnet2/enh/loss/wrapper/pit_solver.py Args: - inf (torch.Tensor): Inferred output tensor. - Shape: (batch, num_inf, ...) - inf_lens (torch.Tensor): Lengths of inferred outputs. - Shape: (batch, num_inf) - ref (torch.Tensor): Reference tensor. - Shape: (batch, num_ref, ...) - ref_lens (torch.Tensor): Lengths of reference outputs. - Shape: (batch, num_ref) - others (Dict, optional): Additional arguments to be passed to the criterion function. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - The mean of the minimum losses across the batch (scalar). - - The optimal permutation for each item in the batch. - Shape: (batch_size, num_ref) - - Raises: - AssertionError: If the shapes of input tensors are inconsistent or - if num_ref doesn't match the number of speakers in the inputs. - - Note: - This method assumes that the number of inferred speakers (num_inf) is equal - to the number of reference speakers (num_ref). - - Examples: - >>> pit_wrapper = PITLossWrapper(criterion_fn, num_ref=2) - >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) - >>> ref = torch.randn(4, 2, 10) # (batch, num_speakers, features) - >>> inf_lens = torch.full((4, 2), 10) - >>> ref_lens = torch.full((4, 2), 10) - >>> loss, perm = pit_wrapper.forward(inf, inf_lens, ref, ref_lens) + inf: Iterable[torch.Tensor], (batch, num_inf, ...) + inf_lens: Iterable[torch.Tensor], (batch, num_inf, ...) + ref: Iterable[torch.Tensor], (batch, num_ref, ...) + ref_lens: Iterable[torch.Tensor], (batch, num_ref, ...) + permute_inf: If true, permute the inference and inference_lens according to + the optimal permutation. """ assert ( self.num_ref @@ -157,37 +99,6 @@ def pair_loss(permutation): @classmethod def permutate(self, perm, *args): - """ - Permute the input tensors according to the given permutation. - - This class method applies the optimal permutation to the input tensors, - rearranging the speaker order for each item in the batch. - - Args: - perm (torch.Tensor): The permutation tensor to apply. - Shape: (batch_size, num_ref) - *args: Variable length argument list of tensors to be permuted. - Each tensor should have shape (batch, num_inf, ...) - - Returns: - List[torch.Tensor]: A list of permuted tensors, each with the same shape - as its corresponding input tensor. - - Raises: - AssertionError: If the batch size or number of speakers is inconsistent - across the input tensors. - - Note: - This method is typically used after finding the optimal permutation - with the forward method to reorder the input tensors accordingly. - - Examples: - >>> pit_wrapper = PITLossWrapper(criterion_fn, num_ref=2) - >>> inf = torch.randn(4, 2, 10) # (batch, num_speakers, features) - >>> inf_lens = torch.full((4, 2), 10) - >>> perm = torch.tensor([[0, 1], [1, 0], [0, 1], [1, 0]]) - >>> permuted_inf, permuted_inf_lens = PITLossWrapper.permutate(perm, inf, inf_lens) - """ ret = [] batch_size = None num_ref = None @@ -208,80 +119,7 @@ def permutate(self, perm, *args): class ESPnetASRModel(SingleESPnetASRModel): - """ - CTC-attention hybrid Encoder-Decoder model for multi-speaker ASR. - - This class extends the SingleESPnetASRModel to support multi-speaker automatic speech - recognition using Permutation Invariant Training (PIT). It combines CTC and - attention-based approaches for improved ASR performance in scenarios with multiple speakers. - - Attributes: - num_inf (int): Number of inferences (outputs) from the model. - num_ref (int): Number of references (ground truth sequences). - pit_ctc (PITLossWrapper): PIT wrapper for the CTC loss. - - Args: - vocab_size (int): Size of the vocabulary. - token_list (Union[Tuple[str, ...], List[str]]): List of tokens. - frontend (Optional[AbsFrontend]): Frontend processing module. - specaug (Optional[AbsSpecAug]): SpecAugment module. - normalize (Optional[AbsNormalize]): Normalization module. - preencoder (Optional[AbsPreEncoder]): Pre-encoder module. - encoder (AbsEncoder): Encoder module. - postencoder (Optional[AbsPostEncoder]): Post-encoder module. - decoder (Optional[AbsDecoder]): Decoder module. - ctc (CTC): CTC module. - joint_network (Optional[torch.nn.Module]): Joint network for transducer-based models. - ctc_weight (float): Weight of CTC loss (0.0 < ctc_weight <= 1.0). - interctc_weight (float): Weight of intermediate CTC loss (must be 0.0 for multi-speaker ASR). - ignore_id (int): Padding value for the ignore index. - lsm_weight (float): Label smoothing weight. - length_normalized_loss (bool): Whether to normalize loss by length. - report_cer (bool): Whether to report Character Error Rate. - report_wer (bool): Whether to report Word Error Rate. - sym_space (str): Space symbol. - sym_blank (str): Blank symbol. - sym_sos (str): Start of sequence symbol. - sym_eos (str): End of sequence symbol. - extract_feats_in_collect_stats (bool): Whether to extract features in collect_stats. - lang_token_id (int): Language token ID. - num_inf (int): Number of inferences (outputs) from the model. - num_ref (int): Number of references (ground truth sequences). - - Note: - - This model requires that num_inf == num_ref. - - The interctc_weight must be set to 0.0 as intermediate CTC is not supported for multi-speaker ASR. - - Examples: - >>> model = ESPnetASRModel( - ... vocab_size=1000, - ... token_list=["", "", "a", "b", "c", ...], - ... frontend=frontend, - ... specaug=specaug, - ... normalize=normalize, - ... preencoder=preencoder, - ... encoder=encoder, - ... postencoder=postencoder, - ... decoder=decoder, - ... ctc=ctc, - ... joint_network=None, - ... ctc_weight=0.3, - ... interctc_weight=0.0, - ... ignore_id=-1, - ... lsm_weight=0.1, - ... length_normalized_loss=False, - ... report_cer=True, - ... report_wer=True, - ... sym_space="", - ... sym_blank="", - ... sym_sos="", - ... sym_eos="", - ... extract_feats_in_collect_stats=True, - ... lang_token_id=-1, - ... num_inf=2, - ... num_ref=2 - ... ) - """ + """CTC-attention hybrid Encoder-Decoder model""" @typechecked def __init__( @@ -361,58 +199,14 @@ def forward( text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """ - Forward pass of the ESPnetASRModel for multi-speaker ASR. - - This method processes the input speech and computes the loss for multi-speaker - automatic speech recognition using a combination of CTC and attention-based approaches. + """Frontend + Encoder + Decoder + Calc loss Args: - speech (torch.Tensor): Input speech tensor (Batch, Length, ...). - speech_lengths (torch.Tensor): Lengths of input speech sequences (Batch,). - text (torch.Tensor): Target text tensor for the first speaker (Batch, Length). - text_lengths (torch.Tensor): Lengths of target text sequences (Batch,). - **kwargs: Additional keyword arguments. - Expected to contain "text_spk{n}" and "text_spk{n}_lengths" for n = 2 to num_ref, - representing the text and text lengths for additional speakers. - - Returns: - Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - - loss: The total loss for the batch. - - stats: A dictionary containing various statistics and metrics: - - loss: Total loss (detached). - - loss_att: Attention loss (if applicable, detached). - - loss_ctc: CTC loss (if applicable, detached). - - loss_transducer: Transducer loss (if applicable, detached). - - acc: Attention accuracy (if applicable). - - cer: Character Error Rate (if applicable). - - wer: Word Error Rate (if applicable). - - cer_ctc: Character Error Rate for CTC (if applicable). - - cer_transducer: Character Error Rate for Transducer (if applicable). - - wer_transducer: Word Error Rate for Transducer (if applicable). - - weight: Batch size (used for averaging). - - Raises: - AssertionError: If the input tensor dimensions are inconsistent or if the - batch sizes don't match across inputs. - - Note: - - This method handles both CTC and attention-based (and potentially transducer-based) ASR approaches. - - It uses Permutation Invariant Training (PIT) for handling multiple speakers. - - The method expects additional text inputs for each speaker beyond the first, - which should be provided in the kwargs as "text_spk{n}" and "text_spk{n}_lengths". - - Examples: - >>> speech = torch.randn(2, 1000, 80) # (batch, time, features) - >>> speech_lengths = torch.tensor([1000, 800]) - >>> text = torch.randint(0, 100, (2, 50)) # (batch, max_text_len) - >>> text_lengths = torch.tensor([45, 30]) - >>> text_spk2 = torch.randint(0, 100, (2, 40)) - >>> text_spk2_lengths = torch.tensor([35, 25]) - >>> loss, stats, weight = model.forward( - ... speech, speech_lengths, text, text_lengths, - ... text_spk2=text_spk2, text_spk2_lengths=text_spk2_lengths - ... ) + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + kwargs: "utt_id" is among the input. """ assert text_lengths.dim() == 1, text_lengths.shape # Check that batch_size is unified diff --git a/espnet2/asr/postencoder/abs_postencoder.py b/espnet2/asr/postencoder/abs_postencoder.py index 8ad063e13a6..cebfa3b7021 100644 --- a/espnet2/asr/postencoder/abs_postencoder.py +++ b/espnet2/asr/postencoder/abs_postencoder.py @@ -5,78 +5,12 @@ class AbsPostEncoder(torch.nn.Module, ABC): - """ - Abstract base class for post-encoder modules in a neural network. - - This class defines the interface for post-encoder modules that process - encoded input sequences. It inherits from both torch.nn.Module and ABC - (Abstract Base Class). - - Attributes: - None - - Note: - Subclasses must implement the `output_size` and `forward` methods. - - Example: - class CustomPostEncoder(AbsPostEncoder): - def output_size(self) -> int: - return 256 - - def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - # Custom implementation - pass - """ - @abstractmethod def output_size(self) -> int: - """ - Returns the output size of the post-encoder. - - This abstract method should be implemented by subclasses to specify - the size of the output tensor produced by the post-encoder. - - Returns: - int: The size of the output tensor. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Example: - class CustomPostEncoder(AbsPostEncoder): - def output_size(self) -> int: - return 256 # Example output size - """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Processes the input tensor through the post-encoder. - - This abstract method should be implemented by subclasses to define the - forward pass of the post-encoder. - - Args: - input (torch.Tensor): The input tensor to be processed. - input_lengths (torch.Tensor): The lengths of each sequence in the input tensor. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - The processed output tensor. - - The updated lengths of each sequence in the output tensor. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Example: - class CustomPostEncoder(AbsPostEncoder): - def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - # Example implementation - processed_output = some_processing(input) - updated_lengths = input_lengths # Or modify if needed - return processed_output, updated_lengths - """ raise NotImplementedError diff --git a/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py b/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py index 4478911f634..75e88a23688 100644 --- a/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py +++ b/espnet2/asr/postencoder/hugging_face_transformers_postencoder.py @@ -24,33 +24,7 @@ class HuggingFaceTransformersPostEncoder(AbsPostEncoder): - """ - Hugging Face Transformers PostEncoder. - - This class implements a post-encoder using Hugging Face Transformers models. - It can be used to process the output of an encoder in a speech recognition - or other sequence processing task. - - Attributes: - transformer (transformers.PreTrainedModel): The Hugging Face transformer model. - lang_token_embed (torch.Tensor): Language token embedding, if applicable. - linear_in (torch.nn.Linear): Linear layer to project input to transformer dimension. - length_adaptor (torch.nn.Sequential): Sequence of layers for length adaptation. - length_adaptor_ratio (int): Ratio of length reduction in the adaptor. - - Args: - input_size (int): Size of the input features. - model_name_or_path (str): Name or path of the pre-trained Hugging Face model. - length_adaptor_n_layers (int, optional): Number of layers in the length adaptor. Defaults to 0. - lang_token_id (int, optional): ID of the language token. Defaults to -1. - - Raises: - ImportError: If the 'transformers' library is not installed. - - Note: - This class requires the 'transformers' library to be installed. - The length adaptor is implemented as described in https://aclanthology.org/2021.acl-long.68.pdf - """ + """Hugging Face Transformers PostEncoder.""" @typechecked def __init__( @@ -153,36 +127,7 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the HuggingFaceTransformersPostEncoder. - - This method processes the input through the length adaptor, linear projection, - and the Hugging Face transformer model. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, time, feature_dim). - input_lengths (torch.Tensor): Tensor of input lengths for each sequence in the batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - output (torch.Tensor): The output of the transformer model. - - input_lengths (torch.Tensor): Updated lengths after processing. - - Raises: - TooShortUttError: If the input sequence is too short for subsampling. - - Note: - - The input is first processed by the length adaptor, which may reduce its temporal dimension. - - If a language token is specified, it's prepended to the input. - - The method handles different configurations of transformer models, including - encoder-decoder models, XLNet, T5, and GPT-2. - - Examples: - >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") - >>> input_tensor = torch.randn(32, 100, 256) # (batch_size, time, feature_dim) - >>> input_lengths = torch.full((32,), 100) - >>> output, output_lengths = encoder.forward(input_tensor, input_lengths) - """ + """Forward.""" if input.size(1) < self.length_adaptor_ratio: raise TooShortUttError( f"has {input.size(1)} frames and is too short for subsampling " @@ -233,40 +178,11 @@ def forward( return output, input_lengths def reload_pretrained_parameters(self): - """ - Reload the pretrained parameters of the transformer model. - - This method resets the transformer's parameters to their original pretrained values. - It's useful for resetting the model to its initial state, especially after fine-tuning. - - Note: - This method logs an info message upon successful reloading of parameters. - - Examples: - >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") - >>> # After some training or parameter updates - >>> encoder.reload_pretrained_parameters() - # This will reset the transformer's parameters to their original pretrained values - """ + self.transformer.load_state_dict(self.pretrained_params) logging.info("Pretrained Transformers model parameters reloaded!") def output_size(self) -> int: - """ - Get the output size of the transformer model. - - Returns: - int: The size of the output features from the transformer model. - - Note: - This method returns the hidden size of the transformer model, which - corresponds to the dimensionality of the output features. - - Examples: - >>> encoder = HuggingFaceTransformersPostEncoder(input_size=256, model_name_or_path="bert-base-uncased") - >>> output_dim = encoder.output_size() - >>> print(output_dim) - 768 # For BERT base model - """ + """Get the output size.""" return self.transformer.config.hidden_size diff --git a/espnet2/asr/postencoder/length_adaptor_postencoder.py b/espnet2/asr/postencoder/length_adaptor_postencoder.py index 1658d0dd5aa..40420197c60 100644 --- a/espnet2/asr/postencoder/length_adaptor_postencoder.py +++ b/espnet2/asr/postencoder/length_adaptor_postencoder.py @@ -14,24 +14,7 @@ class LengthAdaptorPostEncoder(AbsPostEncoder): - """ - Length Adaptor PostEncoder for adjusting the length of input sequences. - - This class implements a Length Adaptor PostEncoder, which is designed to modify - the length of input sequences through a series of convolutional layers. It can - optionally include an initial linear layer for input embedding. - - Attributes: - embed (torch.nn.Sequential or None): Optional input embedding layer. - length_adaptor (torch.nn.Sequential): Sequence of convolutional layers for length adaptation. - length_adaptor_ratio (int): The ratio by which the input length is reduced. - return_int_enc (bool): Flag to determine if intermediate encodings should be returned. - out_sz (int): The output size of the encoder. - - Note: - This implementation is based on the Length Adaptor described in - https://aclanthology.org/2021.acl-long.68.pdf - """ + """Length Adaptor PostEncoder.""" @typechecked def __init__( @@ -76,38 +59,7 @@ def __init__( def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the Length Adaptor PostEncoder. - - This method processes the input tensor through the length adaptor layers, - adjusting its length according to the configured length_adaptor_ratio. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, time_steps, features). - input_lengths (torch.Tensor): Tensor containing the lengths of each sequence in the batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - output tensor of shape (batch_size, adjusted_time_steps, features) - - tensor of adjusted input lengths - - Raises: - TooShortUttError: If the input sequence is too short for the configured subsampling. - - Examples: - >>> encoder = LengthAdaptorPostEncoder(input_size=256, length_adaptor_n_layers=2) - >>> input_tensor = torch.randn(32, 100, 256) - >>> input_lengths = torch.full((32,), 100) - >>> output, output_lengths = encoder(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 25, 256]) - >>> print(output_lengths) - tensor([25, 25, 25, ..., 25, 25, 25]) - - Note: - The input tensor is permuted before and after passing through the length adaptor - to match the expected input format of Conv1d layers. - """ + """Forward.""" if input.size(1) < self.length_adaptor_ratio: raise TooShortUttError( f"has {input.size(1)} frames and is too short for subsampling " @@ -131,25 +83,5 @@ def forward( return input, input_lengths def output_size(self) -> int: - """ - Get the output size of the Length Adaptor PostEncoder. - - Returns: - int: The size of the output features. - - Note: - This method returns the value stored in the `out_sz` attribute, which is set - during the initialization of the LengthAdaptorPostEncoder. It represents - either the original input size or the size after the optional linear embedding, - depending on the configuration. - - Examples: - >>> encoder = LengthAdaptorPostEncoder(input_size=256, output_size=512) - >>> print(encoder.output_size()) - 512 - - >>> encoder = LengthAdaptorPostEncoder(input_size=256) - >>> print(encoder.output_size()) - 256 - """ + """Get the output size.""" return self.out_sz diff --git a/espnet2/asr/preencoder/abs_preencoder.py b/espnet2/asr/preencoder/abs_preencoder.py index 27c5097db57..67777477e0b 100644 --- a/espnet2/asr/preencoder/abs_preencoder.py +++ b/espnet2/asr/preencoder/abs_preencoder.py @@ -5,95 +5,12 @@ class AbsPreEncoder(torch.nn.Module, ABC): - """ - Abstract base class for pre-encoder modules in a neural network. - - This class defines the interface for pre-encoder modules, which are typically - used to process input data before it's passed to the main encoder in a - neural network architecture. - - Attributes: - None - - Examples: - ```python - class MyPreEncoder(AbsPreEncoder): - def output_size(self) -> int: - return 256 - - def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - # Implementation of forward pass - pass - - pre_encoder = MyPreEncoder() - ``` - - Note: - Subclasses must implement the `output_size` and `forward` methods. - """ - @abstractmethod def output_size(self) -> int: - """ - Returns the output size of the pre-encoder. - - This method should be implemented by subclasses to specify the - dimensionality of the output produced by the pre-encoder. - - Returns: - int: The size of the output tensor produced by the pre-encoder. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Examples: - ```python - class MyPreEncoder(AbsPreEncoder): - def output_size(self) -> int: - return 256 - - pre_encoder = MyPreEncoder() - output_dim = pre_encoder.output_size() # Returns 256 - ``` - """ raise NotImplementedError @abstractmethod def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Performs the forward pass of the pre-encoder. - - This method should be implemented by subclasses to define the forward - computation of the pre-encoder. - - Args: - input (torch.Tensor): The input tensor to be processed. - input_lengths (torch.Tensor): A tensor containing the lengths of each - sequence in the input batch. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - The output tensor after pre-encoding. - - A tensor of output lengths corresponding to each sequence in the batch. - - Raises: - NotImplementedError: If the method is not implemented by a subclass. - - Examples: - ```python - class MyPreEncoder(AbsPreEncoder): - def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - # Example implementation - output = self.some_processing(input) - output_lengths = input_lengths # Assuming no change in sequence length - return output, output_lengths - - pre_encoder = MyPreEncoder() - input_tensor = torch.randn(32, 100, 80) # Batch size 32, 100 time steps, 80 features - input_lengths = torch.full((32,), 100) # All sequences in batch have length 100 - output, output_lengths = pre_encoder(input_tensor, input_lengths) - ``` - """ raise NotImplementedError diff --git a/espnet2/asr/preencoder/linear.py b/espnet2/asr/preencoder/linear.py index 2c56d6720ba..94e857bdb12 100644 --- a/espnet2/asr/preencoder/linear.py +++ b/espnet2/asr/preencoder/linear.py @@ -13,35 +13,7 @@ class LinearProjection(AbsPreEncoder): - """ - Linear Projection Preencoder. - - This class implements a linear projection preencoder that applies a linear transformation - followed by dropout to the input features. It is a subclass of AbsPreEncoder. - - Attributes: - output_dim (int): The dimension of the output features. - linear_out (torch.nn.Linear): The linear transformation layer. - dropout (torch.nn.Dropout): The dropout layer. - - Args: - input_size (int): The size of the input features. - output_size (int): The size of the output features. - dropout (float, optional): The dropout rate. Defaults to 0.0. - - Example: - >>> import torch - >>> preencoder = LinearProjection(input_size=100, output_size=80, dropout=0.1) - >>> input_tensor = torch.randn(32, 10, 100) # (batch_size, sequence_length, input_size) - >>> input_lengths = torch.full((32,), 10) - >>> output, output_lengths = preencoder(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 10, 80]) - - Note: - This preencoder does not modify the input lengths, so the output_lengths - will be the same as the input_lengths. - """ + """Linear Projection Preencoder.""" @typechecked def __init__(self, input_size: int, output_size: int, dropout: float = 0.0): @@ -55,52 +27,10 @@ def __init__(self, input_size: int, output_size: int, dropout: float = 0.0): def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass of the LinearProjection preencoder. - - This method applies dropout to the input, then performs a linear transformation. - - Args: - input (torch.Tensor): Input tensor of shape (batch_size, sequence_length, input_size). - input_lengths (torch.Tensor): Tensor of input sequence lengths of shape (batch_size,). - - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - output (torch.Tensor): The transformed output tensor of shape - (batch_size, sequence_length, output_size). - - input_lengths (torch.Tensor): The input sequence lengths, unchanged. - - Note: - This method does not modify the input lengths, so the returned input_lengths - are the same as the input. - - Example: - >>> preencoder = LinearProjection(input_size=100, output_size=80) - >>> input_tensor = torch.randn(32, 10, 100) - >>> input_lengths = torch.full((32,), 10) - >>> output, output_lengths = preencoder.forward(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 10, 80]) - >>> print(torch.all(output_lengths == input_lengths)) - True - """ + """Forward.""" output = self.linear_out(self.dropout(input)) return output, input_lengths # no state in this layer def output_size(self) -> int: - """ - Get the output size of the LinearProjection preencoder. - - Returns: - int: The dimension of the output features. - - Example: - >>> preencoder = LinearProjection(input_size=100, output_size=80) - >>> print(preencoder.output_size()) - 80 - - Note: - This method returns the value of the `output_dim` attribute, which is set - during the initialization of the LinearProjection instance. - """ + """Get the output size.""" return self.output_dim diff --git a/espnet2/asr/preencoder/sinc.py b/espnet2/asr/preencoder/sinc.py index 7e0da592002..778ccf8acfe 100644 --- a/espnet2/asr/preencoder/sinc.py +++ b/espnet2/asr/preencoder/sinc.py @@ -16,37 +16,26 @@ class LightweightSincConvs(AbsPreEncoder): - """ - Lightweight Sinc Convolutions for end-to-end speech recognition from raw audio. + """Lightweight Sinc Convolutions. - This class implements the Lightweight Sinc Convolutions as described in - "Lightweight End-to-End Speech Recognition from Raw Audio Data Using - Sinc-Convolutions" by Kürzinger et al. It processes raw audio input - instead of using precomputed features, serving as a pre-encoder in the - speech recognition pipeline. + Instead of using precomputed features, end-to-end speech recognition + can also be done directly from raw audio using sinc convolutions, as + described in "Lightweight End-to-End Speech Recognition from Raw Audio + Data Using Sinc-Convolutions" by Kürzinger et al. + https://arxiv.org/abs/2010.07597 - To use this module, set it as the pre-encoder with `preencoder: sinc` - and use the sliding window frontend with `frontend: sliding_window` in your - YAML configuration file. + To use Sinc convolutions in your model instead of the default f-bank + frontend, set this module as your pre-encoder with `preencoder: sinc` + and use the input of the sliding window frontend with + `frontend: sliding_window` in your yaml configuration file. + So that the process flow is: - The process flow should be: Frontend (SlidingWindow) -> SpecAug -> Normalization -> Pre-encoder (LightweightSincConvs) -> Encoder -> Decoder - This method performs data augmentation in the time domain, as opposed to - the spectral domain in the default frontend. - - Attributes: - fs (int): Sample rate of the input audio. - in_channels (int): Number of input channels. - out_channels (int): Number of output channels (for each input channel). - activation_type (str): Type of activation function used. - dropout_type (str): Type of dropout function used. - windowing_type (str): Type of windowing function used. - scale_type (str): Type of filter-bank initialization scale. - - Note: - Use `plot_sinc_filters.py` to visualize the learned Sinc filters. + Note that this method also performs data augmentation in time domain + (vs. in spectral domain in the default frontend). + Use `plot_sinc_filters.py` to visualize the learned Sinc filters. """ @typechecked @@ -177,30 +166,24 @@ def gen_lsc_block( dropout_probability: float = 0.15, avgpool=False, ): - """ - Generate a convolutional block for Lightweight Sinc convolutions. + """Generate a convolutional block for Lightweight Sinc convolutions. Each block consists of either a depthwise or a depthwise-separable - convolution together with dropout, (batch-)normalization layer, and + convolutions together with dropout, (batch-)normalization layer, and an optional average-pooling layer. Args: - in_channels (int): Number of input channels. - out_channels (int): Number of output channels. - depthwise_kernel_size (int): Kernel size of the depthwise convolution. Defaults to 9. - depthwise_stride (int): Stride of the depthwise convolution. Defaults to 1. - depthwise_groups (int, optional): Number of groups of the depthwise convolution. - If None, it's set to the GCD of in_channels and out_channels. - pointwise_groups (int): Number of groups of the pointwise convolution. Defaults to 0. - dropout_probability (float): Dropout probability in the block. Defaults to 0.15. - avgpool (bool): If True, an AvgPool layer is inserted. Defaults to False. + in_channels: Number of input channels. + out_channels: Number of output channels. + depthwise_kernel_size: Kernel size of the depthwise convolution. + depthwise_stride: Stride of the depthwise convolution. + depthwise_groups: Number of groups of the depthwise convolution. + pointwise_groups: Number of groups of the pointwise convolution. + dropout_probability: Dropout probability in the block. + avgpool: If True, an AvgPool layer is inserted. Returns: - torch.nn.Sequential: Neural network building block for Lightweight Sinc convolutions. - - Note: - The block structure adapts based on the provided parameters, allowing for - flexible configuration of the convolutional layers. + torch.nn.Sequential: Neural network building block. """ block = OrderedDict() if not depthwise_groups: @@ -227,21 +210,7 @@ def gen_lsc_block( return torch.nn.Sequential(block) def espnet_initialization_fn(self): - """ - Initialize sinc filters and batch normalization layers. - - This method initializes the sinc filters with filterbank values and sets - the initial weights and biases for batch normalization layers in the network. - - The initialization process includes: - 1. Initializing the sinc filters using the `init_filters` method. - 2. Setting the weight data of all BatchNorm1d layers to 1.0. - 3. Setting the bias data of all BatchNorm1d layers to 0.0. - - Note: - This initialization is specific to the Lightweight Sinc Convolutions - architecture and helps in achieving better initial performance. - """ + """Initialize sinc filters with filterbank values.""" self.filters.init_filters() for block in self.blocks: for layer in block: @@ -252,35 +221,18 @@ def espnet_initialization_fn(self): def forward( self, input: torch.Tensor, input_lengths: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Apply Lightweight Sinc Convolutions to the input tensor. + """Apply Lightweight Sinc Convolutions. - Args: - input (torch.Tensor): Input tensor of shape (B, T, C_in, D_in), where: - B is the batch size, - T is the time dimension, - C_in is the number of input channels, - D_in is the input feature dimension. - input_lengths (torch.Tensor): Tensor containing the lengths of each sequence in the batch. + The input shall be formatted as (B, T, C_in, D_in) + with B as batch size, T as time dimension, C_in as channels, + and D_in as feature dimension. - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - output_frames (torch.Tensor): Output tensor of shape (B, T, C_out*D_out), where: - C_out is the number of output channels, - D_out is the output feature dimension. - - input_lengths (torch.Tensor): The input lengths tensor (unchanged). - - Note: - The current module structure only handles D_in=400, resulting in D_out=1. - For the multichannel case, C_out is the number of out_channels given at - initialization multiplied by C_in. - - Example: - >>> input_tensor = torch.randn(32, 1000, 1, 400) # (B, T, C_in, D_in) - >>> input_lengths = torch.full((32,), 1000) - >>> output, output_lengths = self.forward(input_tensor, input_lengths) - >>> print(output.shape) - torch.Size([32, 1000, 256]) # Assuming out_channels=256 + The output will then be (B, T, C_out*D_out) + with C_out and D_out as output dimensions. + + The current module structure only handles D_in=400, so that D_out=1. + Remark for the multichannel case: C_out is the number of out_channels + given at initialization multiplied with C_in. """ # Transform input data: # (B, T, C_in, D_in) -> (B*T, C_in, D_in) @@ -294,37 +246,14 @@ def forward( return output_frames, input_lengths # no state in this layer def output_size(self) -> int: - """ - Get the output size of the Lightweight Sinc Convolutions. - - Returns: - int: The total number of output features, calculated as the product - of the number of output channels and the number of input channels. - - Note: - This method is useful for determining the size of the output tensor - produced by the Lightweight Sinc Convolutions, which can be helpful - when connecting this module to subsequent layers in the network. - """ + """Get the output size.""" return self.out_channels * self.in_channels class SpatialDropout(torch.nn.Module): - """ - Spatial dropout module for applying dropout to full channels. - - This module applies dropout to entire channels on tensors of input shape (B, C, D), - where B is the batch size, C is the number of channels, and D is the feature dimension. - It's designed to maintain spatial coherence by dropping out entire feature maps - instead of individual elements. + """Spatial dropout module. - Attributes: - dropout (torch.nn.Dropout2d): The underlying dropout layer. - shape (tuple): The shape used for permuting the input tensor. - - Note: - This implementation is particularly useful for 1D convolutional neural networks - where you want to drop entire channels rather than random elements. + Apply dropout to full channels on tensors of input (B, C, D) """ @typechecked @@ -346,25 +275,7 @@ def __init__( self.shape = (shape,) def forward(self, x: torch.Tensor) -> torch.Tensor: - """ - Apply spatial dropout to the input tensor. - - This method applies dropout to entire channels of the input tensor. - It first permutes the input tensor according to the specified shape, - applies dropout, and then permutes the tensor back to its original shape. - - Args: - x (torch.Tensor): Input tensor of shape (B, C, D), where B is the batch size, - C is the number of channels, and D is the feature dimension. - - Returns: - torch.Tensor: Output tensor with spatial dropout applied, maintaining the - same shape as the input tensor. - - Note: - The dropout is applied to entire channels, preserving spatial coherence - within each channel. - """ + """Forward of spatial dropout module.""" y = x.permute(*self.shape) y = self.dropout(y) return y.permute(*self.shape) diff --git a/espnet2/asr/specaug/abs_specaug.py b/espnet2/asr/specaug/abs_specaug.py index 95e3df0d309..6c9c6d8ea18 100644 --- a/espnet2/asr/specaug/abs_specaug.py +++ b/espnet2/asr/specaug/abs_specaug.py @@ -4,62 +4,14 @@ class AbsSpecAug(torch.nn.Module): - """ - Abstract base class for spectrogram augmentation. - - This class defines the interface for spectrogram augmentation modules in a - neural network pipeline. Spectrogram augmentation is typically applied after - the frontend processing and before normalization, encoding, and decoding steps. + """Abstract class for the augmentation of spectrogram - Attributes: - None + The process-flow: - Note: - Subclasses must implement the `forward` method to define specific - augmentation techniques. - - Examples: - >>> class MySpecAug(AbsSpecAug): - ... def forward(self, x, x_lengths=None): - ... # Implement augmentation logic here - ... return augmented_x, x_lengths - ... - >>> spec_aug = MySpecAug() - >>> augmented_spec, lengths = spec_aug(input_spec, input_lengths) + Frontend -> SpecAug -> Normalization -> Encoder -> Decoder """ def forward( self, x: torch.Tensor, x_lengths: torch.Tensor = None ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """ - Apply spectrogram augmentation to the input tensor. - - This method should be implemented by subclasses to perform specific - augmentation techniques on the input spectrogram. - - Args: - x (torch.Tensor): Input spectrogram tensor. - x_lengths (torch.Tensor, optional): Tensor containing the lengths of each - sequence in the batch. Defaults to None. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: - - The augmented spectrogram tensor. - - The updated lengths tensor (if x_lengths was provided, otherwise None). - - Raises: - NotImplementedError: This method must be implemented by subclasses. - - Examples: - >>> class MySpecAug(AbsSpecAug): - ... def forward(self, x, x_lengths=None): - ... # Apply augmentation (e.g., time warping, frequency masking) - ... augmented_x = some_augmentation_function(x) - ... return augmented_x, x_lengths - ... - >>> spec_aug = MySpecAug() - >>> input_spec = torch.randn(32, 80, 100) # (batch_size, num_mels, time_steps) - >>> input_lengths = torch.full((32,), 100) - >>> augmented_spec, updated_lengths = spec_aug(input_spec, input_lengths) - """ raise NotImplementedError diff --git a/espnet2/asr/specaug/specaug.py b/espnet2/asr/specaug/specaug.py index b3692c971be..8dde0f53b1f 100644 --- a/espnet2/asr/specaug/specaug.py +++ b/espnet2/asr/specaug/specaug.py @@ -8,44 +8,17 @@ class SpecAug(AbsSpecAug): - """ - Implementation of SpecAugment for speech augmentation. - - This class applies various augmentation techniques to speech spectrograms, - including time warping, frequency masking, and time masking. - - Attributes: - apply_time_warp (bool): Whether to apply time warping. - apply_freq_mask (bool): Whether to apply frequency masking. - apply_time_mask (bool): Whether to apply time masking. - time_warp (TimeWarp): Time warping object if enabled, else None. - freq_mask (MaskAlongAxis): Frequency masking object if enabled, else None. - time_mask (Union[MaskAlongAxis, MaskAlongAxisVariableMaxWidth]): Time masking object if enabled, else None. - - Args: - apply_time_warp (bool): Whether to apply time warping. Defaults to True. - time_warp_window (int): Window size for time warping. Defaults to 5. - time_warp_mode (str): Interpolation mode for time warping. Defaults to "bicubic". - apply_freq_mask (bool): Whether to apply frequency masking. Defaults to True. - freq_mask_width_range (Union[int, Sequence[int]]): Range of width for frequency masks. Defaults to (0, 20). - num_freq_mask (int): Number of frequency masks to apply. Defaults to 2. - apply_time_mask (bool): Whether to apply time masking. Defaults to True. - time_mask_width_range (Optional[Union[int, Sequence[int]]]): Range of width for time masks. Defaults to None. - time_mask_width_ratio_range (Optional[Union[float, Sequence[float]]]): Range of width ratio for time masks. Defaults to None. - num_time_mask (int): Number of time masks to apply. Defaults to 2. + """Implementation of SpecAug. - Raises: - ValueError: If no augmentation technique is applied or if both time_mask_width_range - and time_mask_width_ratio_range are specified. + Reference: + Daniel S. Park et al. + "SpecAugment: A Simple Data + Augmentation Method for Automatic Speech Recognition" - Note: - This implementation is based on the SpecAugment paper by Daniel S. Park et al. - When using CUDA mode, time warping may not be reproducible due to - `torch.nn.functional.interpolate`. + .. warning:: + When using cuda mode, time_warp doesn't have reproducibility + due to `torch.nn.functional.interpolate`. - Examples: - >>> specaug = SpecAug(apply_time_warp=True, apply_freq_mask=True, apply_time_mask=True) - >>> augmented_spec, lengths = specaug(input_spec, input_lengths) """ def __init__( @@ -119,27 +92,6 @@ def __init__( self.time_mask = None def forward(self, x, x_lengths=None): - """ - Apply SpecAugment to the input spectrogram. - - This method applies the configured augmentation techniques (time warping, - frequency masking, and time masking) to the input spectrogram. - - Args: - x (Tensor): Input spectrogram tensor of shape (batch_size, num_channels, num_freq, num_time). - x_lengths (Tensor, optional): Lengths of each sequence in the batch. Defaults to None. - - Returns: - Tuple[Tensor, Tensor]: - - Augmented spectrogram tensor of the same shape as the input. - - Updated lengths tensor (if x_lengths was provided, otherwise None). - - Examples: - >>> specaug = SpecAug() - >>> input_spec = torch.randn(32, 1, 80, 100) # (batch_size, channels, freq, time) - >>> input_lengths = torch.full((32,), 100) - >>> augmented_spec, augmented_lengths = specaug.forward(input_spec, input_lengths) - """ if self.time_warp is not None: x, x_lengths = self.time_warp(x, x_lengths) if self.freq_mask is not None: diff --git a/espnet2/asr/state_spaces/attention.py b/espnet2/asr/state_spaces/attention.py index 68ce97b9dd7..70f9579b4ac 100644 --- a/espnet2/asr/state_spaces/attention.py +++ b/espnet2/asr/state_spaces/attention.py @@ -10,43 +10,16 @@ class MultiHeadedAttention(SequenceModule): - """ - Multi-Head Attention layer inheriting from SequenceModule. - - This class implements a Multi-Head Attention mechanism, which allows the model - to jointly attend to information from different representation subspaces at - different positions. It extends the functionality of the default Multi-Head - Attention module in ESPnet by returning an additional dummy state and - providing a step function for autoregressive inference. - - Attributes: - d_k (int): The dimension of the key and query vectors. - h (int): The number of attention heads. - linear_q (nn.Linear): Linear transformation for queries. - linear_k (nn.Linear): Linear transformation for keys. - linear_v (nn.Linear): Linear transformation for values. - linear_out (nn.Linear): Linear transformation for output. - attn (torch.Tensor): Attention weights. - dropout (nn.Dropout): Dropout layer. - d_output (int): The output dimension. + """Multi-Head Attention layer inheriting SequenceModule. + + Comparing default MHA module in ESPnet, this module returns additional dummy state + and has step function for autoregressive inference. Args: - n_feat (int): The number of expected features in the input. - n_head (int): The number of attention heads. - dropout (float, optional): Dropout probability. Defaults to 0.0. - transposed (bool, optional): If True, expects transposed input. Defaults to False. - **kwargs: Additional keyword arguments. - - Note: - This implementation assumes that the dimension of the key (d_k) always - equals the dimension of the value (d_v), and that n_feat is divisible by n_head. - - Example: - >>> mha = MultiHeadedAttention(n_feat=512, n_head=8, dropout=0.1) - >>> query = torch.randn(32, 10, 512) # (batch_size, seq_len, n_feat) - >>> output, _ = mha(query) - >>> output.shape - torch.Size([32, 10, 512]) + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + """ def __init__(self, n_feat, n_head, dropout=0.0, transposed=False, **kwargs): @@ -65,36 +38,18 @@ def __init__(self, n_feat, n_head, dropout=0.0, transposed=False, **kwargs): self.d_output = n_feat def forward_qkv(self, query, key, value): - """ - Transform query, key, and value tensors for multi-head attention. - - This method applies linear transformations to the input query, key, and value - tensors, reshapes them to separate the head dimension, and transposes the - resulting tensors to prepare them for the attention mechanism. + """Transform query, key and value. Args: - query (torch.Tensor): Query tensor of shape (batch_size, time1, size). - key (torch.Tensor): Key tensor of shape (batch_size, time2, size). - value (torch.Tensor): Value tensor of shape (batch_size, time2, size). + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). Returns: - tuple: A tuple containing: - - q (torch.Tensor): Transformed query tensor of shape (batch_size, n_head, time1, d_k). - - k (torch.Tensor): Transformed key tensor of shape (batch_size, n_head, time2, d_k). - - v (torch.Tensor): Transformed value tensor of shape (batch_size, n_head, time2, d_k). - - Note: - The input tensors are expected to have the same size in their last dimension, - which should be equal to n_feat (the number of features) in the MultiHeadedAttention class. - - Example: - >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) - >>> query = torch.randn(32, 10, 512) - >>> key = torch.randn(32, 15, 512) - >>> value = torch.randn(32, 15, 512) - >>> q, k, v = mha.forward_qkv(query, key, value) - >>> q.shape, k.shape, v.shape - (torch.Size([32, 8, 10, 64]), torch.Size([32, 8, 15, 64]), torch.Size([32, 8, 15, 64])) + torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). + torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). + torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). + """ n_batch = query.size(0) q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) @@ -107,37 +62,17 @@ def forward_qkv(self, query, key, value): return q, k, v def forward_attention(self, value, scores, mask): - """ - Compute attention context vector. - - This method applies the attention mechanism to the transformed value tensor - using the provided attention scores and mask. It computes the weighted sum - of values based on the attention distribution. + """Compute attention context vector. Args: - value (torch.Tensor): Transformed value tensor of shape (batch_size, n_head, time2, d_k). - scores (torch.Tensor): Attention score tensor of shape (batch_size, n_head, time1, time2). - mask (torch.Tensor): Mask tensor of shape (batch_size, 1, time2) or (batch_size, time1, time2). - If provided, positions with 1 are masked (i.e., set to -inf before softmax). + value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). + scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). + mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). Returns: - torch.Tensor: Output tensor of shape (batch_size, time1, d_model) containing - the weighted sum of values according to the attention distribution. - - Note: - - The method updates the `self.attn` attribute with the computed attention weights. - - If a mask is provided, it is applied before the softmax to prevent attention - to certain positions. - - Dropout is applied to the attention weights before computing the weighted sum. - - Example: - >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) - >>> value = torch.randn(32, 8, 15, 64) - >>> scores = torch.randn(32, 8, 10, 15) - >>> mask = torch.ones(32, 1, 15).bool() - >>> output = mha.forward_attention(value, scores, mask) - >>> output.shape - torch.Size([32, 10, 512]) + torch.Tensor: Transformed value (#batch, time1, d_model) + weighted by the attention score (#batch, time1, time2). + """ n_batch = value.size(0) if mask is not None: @@ -161,41 +96,18 @@ def forward_attention(self, value, scores, mask): return self.linear_out(x) # (batch, time1, d_model) def forward(self, query, memory=None, mask=None, *args, **kwargs): - """ - Compute scaled dot product attention. - - This method performs the full multi-head attention operation, including the - transformation of inputs, computation of attention scores, and application - of attention weights to produce the final output. + """Compute scaled dot product attention. Args: - query (torch.Tensor): Query tensor of shape (batch_size, time1, size). - memory (torch.Tensor, optional): Memory tensor of shape (batch_size, time2, size). - If None, self-attention is performed using the query as both key and value. - mask (torch.Tensor, optional): Mask tensor of shape (batch_size, 1, time2) or - (batch_size, time1, time2). Positions with 1 are masked. - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor of shape (batch_size, time1, d_model). - - None: A placeholder for compatibility with other modules that return a state. - - Note: - - If memory is None, self-attention is performed using the query as both key and value. - - The attention scores are scaled by 1/sqrt(d_k) before applying the mask and softmax. - - The output is obtained by applying the attention weights to the value vectors - and then passing through a final linear transformation. - - Example: - >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) - >>> query = torch.randn(32, 10, 512) - >>> memory = torch.randn(32, 15, 512) - >>> mask = torch.ones(32, 1, 15).bool() - >>> output, _ = mha.forward(query, memory, mask) - >>> output.shape - torch.Size([32, 10, 512]) + torch.Tensor: Output tensor (#batch, time1, d_model). + """ # self-attention if memory is None: @@ -205,42 +117,6 @@ def forward(self, query, memory=None, mask=None, *args, **kwargs): return self.forward_attention(v, scores, mask), None def step(self, query, state, memory=None, mask=None, **kwargs): - """ - Perform a single step of multi-head attention for autoregressive inference. - - This method is designed for use in autoregressive decoding, where attention is - computed for a single time step. It wraps the forward method to handle the - single-step case efficiently. - - Args: - query (torch.Tensor): Query tensor for the current step, shape (batch_size, 1, size). - state (Any): The previous state (unused in this implementation). - memory (torch.Tensor, optional): Memory tensor of shape (batch_size, time2, size). - If None, self-attention is performed using the query as both key and value. - mask (torch.Tensor, optional): Mask tensor of shape (batch_size, 1, time2) or - (batch_size, 1, time2). Positions with 1 are masked. - **kwargs: Additional keyword arguments passed to the forward method. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor for the current step, shape (batch_size, size). - - Any: The updated state (same as input state in this implementation). - - Note: - - This method assumes that the query is for a single time step. - - The output is squeezed to remove the time dimension, as it's always 1 in this case. - - The state is passed through unchanged, as the current implementation - doesn't utilize persistent state between steps. - - Example: - >>> mha = MultiHeadedAttention(n_feat=512, n_head=8) - >>> query = torch.randn(32, 1, 512) - >>> memory = torch.randn(32, 15, 512) - >>> mask = torch.ones(32, 1, 15).bool() - >>> output, new_state = mha.step(query, None, memory, mask) - >>> output.shape - torch.Size([32, 512]) - """ if memory is None: memory = query return self.forward(query, memory, mask=mask, **kwargs)[0].squeeze(1), state diff --git a/espnet2/asr/state_spaces/base.py b/espnet2/asr/state_spaces/base.py index 92141887577..91513920ead 100644 --- a/espnet2/asr/state_spaces/base.py +++ b/espnet2/asr/state_spaces/base.py @@ -6,84 +6,32 @@ class SequenceModule(nn.Module): - """ - Abstract base class for sequence models. - - This class defines the interface for sequence models that transform input tensors - of shape (n_batch, l_sequence, d_model) to (n_batch, l_sequence, d_output). - - All models inheriting from this class must implement the required methods and - attributes. Optional methods provide additional functionality for recurrent - processing and state management. - - Attributes: - d_model (int): Model dimension (generally same as input dimension). - d_output (int): Output dimension of the model. - - Args: - d_model (int): The input dimension of the model. - transposed (bool, optional): If True, expects input in (n_batch, d_model, l_sequence) - format. Defaults to False. - - Required Methods: - forward(self, x, state=None, **kwargs): Performs the forward pass of the model. - __init__(self, d_model, transposed=False, **kwargs): Initializes the model. + """Abstract sequence model class. - Optional Methods: - default_state(self, *batch_shape, device=None): Creates initial state for a batch. - step(self, x, state=None, **kwargs): Processes one step of the input sequence. - state_to_tensor(self): Returns a function to map state to a single tensor. + All models must adhere to this interface - Properties: - d_state (int): Dimension of the output of self.state_to_tensor. + A SequenceModule is generally a model that transforms an input of shape + (n_batch, l_sequence, d_model) to (n_batch, l_sequence, d_output) - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model, d_output): - super().__init__() - self.d_model = d_model - self.d_output = d_output - self.linear = nn.Linear(d_model, d_output) + REQUIRED methods and attributes + forward, d_model, d_output: controls standard forward pass, + a sequence-to-sequence transformation + __init__ should also satisfy the following interface; + see SequenceIdentity for an example + def __init__(self, d_model, transposed=False, **kwargs) - def forward(self, x, state=None): - return self.linear(x), None - - model = MySequenceModel(d_model=64, d_output=32) - x = torch.randn(16, 100, 64) # (batch, sequence, d_model) - output, _ = model(x) # output shape: (16, 100, 32) - - Note: - Subclasses must set self._d_model and self._d_output in their __init__ method. + OPTIONAL methods + default_state, step: allows stepping the model recurrently with a hidden state + state_to_tensor, d_state: allows decoding from hidden state """ @property def d_model(self): - """ - Model dimension (generally same as input dimension). - - This property is required for all SequenceModule instantiations. It is used by - the rest of the pipeline (e.g., model backbone, encoder) to track the internal - shapes of the full model. - - Returns: - int: The model dimension. + """Model dimension (generally same as input dimension). - Raises: - NotImplementedError: If the SequenceModule instantiation has not set d_model. - - Note: - Subclasses must set self._d_model in their __init__ method. - - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model): - super().__init__() - self._d_model = d_model # Set the internal attribute - - # ... other methods ... - - model = MySequenceModel(64) - print(model.d_model) # Output: 64 + This attribute is required for all SequenceModule instantiations. + It is used by the rest of the pipeline + (e.g. model backbone, encoder) to track the internal shapes of the full model. """ if getattr(self, "_d_model", None) is None: raise NotImplementedError("SequenceModule instantiation must set d_model") @@ -91,65 +39,15 @@ def __init__(self, d_model): @d_model.setter def d_model(self, d): - """ - Model dimension (generally same as input dimension). - - This property is required for all SequenceModule instantiations. It is used by - the rest of the pipeline (e.g., model backbone, encoder) to track the internal - shapes of the full model. - - Returns: - int: The model dimension. - - Raises: - NotImplementedError: If the SequenceModule instantiation has not set d_model. - - Note: - Subclasses must set self._d_model in their __init__ method. - - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model): - super().__init__() - self._d_model = d_model # Set the internal attribute - - # ... other methods ... - - model = MySequenceModel(64) - print(model.d_model) # Output: 64 - """ self._d_model = d @property def d_output(self): - """ - Output dimension of the model. - - This property is required for all SequenceModule instantiations. It is used by - the rest of the pipeline (e.g., model backbone, decoder) to track the internal - shapes of the full model. - - Returns: - int: The output dimension of the model. - - Raises: - NotImplementedError: If the SequenceModule instantiation has not specified - d_output for the decoder. + """Output dimension of model. - Note: - Subclasses must set self._d_output in their __init__ method. - - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model, d_output): - super().__init__() - self._d_model = d_model - self._d_output = d_output # Set the internal attribute - - # ... other methods ... - - model = MySequenceModel(d_model=64, d_output=32) - print(model.d_output) # Output: 32 + This attribute is required for all SequenceModule instantiations. + It is used by the rest of the pipeline + (e.g. model backbone, decoder) to track the internal shapes of the full model. """ if getattr(self, "_d_output", None) is None: raise NotImplementedError( @@ -159,284 +57,58 @@ def __init__(self, d_model, d_output): @d_output.setter def d_output(self, d): - """ - Output dimension of the model. - - This property is required for all SequenceModule instantiations. It is used by - the rest of the pipeline (e.g., model backbone, decoder) to track the internal - shapes of the full model. - - Returns: - int: The output dimension of the model. - - Raises: - NotImplementedError: If the SequenceModule instantiation has not specified - d_output for the decoder. - - Note: - Subclasses must set self._d_output in their __init__ method. + self._d_output = d - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model, d_output): - super().__init__() - self._d_model = d_model - self._d_output = d_output # Set the internal attribute + def forward(self, x, state=None, **kwargs): + """Forward pass. - # ... other methods ... + A sequence-to-sequence transformation with an optional state. - model = MySequenceModel(d_model=64, d_output=32) - print(model.d_output) # Output: 32 - """ - self._d_output = d + Generally, this should map a tensor of shape + (batch, length, self.d_model) to (batch, length, self.d_output) - def forward(self, x, state=None, **kwargs): - """ - Perform the forward pass of the sequence model. - - This method implements a sequence-to-sequence transformation with an optional state. - It should map a tensor of shape (batch, length, self.d_model) to - (batch, length, self.d_output). - - Args: - x (torch.Tensor): Input tensor of shape (batch, length, self.d_model). - state (Any, optional): Initial state for the forward pass. Defaults to None. - **kwargs: Additional keyword arguments. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor of shape (batch, length, self.d_output). - - Any: Updated state. This can be any additional information, such as - the hidden state for RNN and SSM layers, or state information for - certain types of transformer layers (e.g., Transformer-XL). - - Example: - class MySequenceModel(SequenceModule): - def __init__(self, d_model, d_output): - super().__init__() - self.d_model = d_model - self.d_output = d_output - self.linear = nn.Linear(d_model, d_output) - - def forward(self, x, state=None): - output = self.linear(x) - return output, None - - model = MySequenceModel(d_model=64, d_output=32) - x = torch.randn(16, 100, 64) # (batch, length, d_model) - output, new_state = model(x) - print(output.shape) # torch.Size([16, 100, 32]) + Additionally, it returns a "state" which can be any additional information + For example, RNN and SSM layers may return their hidden state, + while some types of transformer layers + (e.g. Transformer-XL) may want to pass a state as well """ return x, None @property def state_to_tensor(self): - """ - Function to map a state to a single tensor. - - This property should return a function that converts the model's state into a - single tensor representation. It is primarily used when the hidden state, rather - than the output sequence, is needed for final prediction. This method is - currently only used in conjunction with the StateDecoder. - - Returns: - Callable: A function that takes a state as input and returns a tensor - representation of that state. If not implemented, returns a lambda - function that always returns None. - - Note: - Subclasses should override this property if they want to provide a custom - state-to-tensor conversion. - - Example: - class MyRNNModel(SequenceModule): - def __init__(self, d_model, d_hidden): - super().__init__() - self.rnn = nn.RNN(d_model, d_hidden) - self._d_state = d_hidden - - @property - def state_to_tensor(self): - return lambda state: state[0].squeeze(0) - - # ... other methods ... - - model = MyRNNModel(d_model=64, d_hidden=32) - x = torch.randn(16, 100, 64) # (batch, length, d_model) - _, final_state = model(x) - state_tensor = model.state_to_tensor(final_state) - print(state_tensor.shape) # torch.Size([16, 32]) + """Return a function mapping a state to a single tensor. + + This method should be implemented if one wants to use + the hidden state insteadof the output sequence for final prediction. + Currently only used with the StateDecoder. """ return lambda _: None @property def d_state(self): - """ - Dimension of the output of self.state_to_tensor. - - This property returns the dimension of the tensor produced by the state_to_tensor - method. It is useful for determining the size of the state representation, - particularly when using the state for downstream tasks or in the StateDecoder. - - Returns: - int or None: The dimension of the state tensor. Returns None if not implemented - or if the state does not have a fixed dimension. - - Note: - Subclasses should override this property if they implement a custom - state_to_tensor method with a known output dimension. - - Example: - class MyLSTMModel(SequenceModule): - def __init__(self, d_model, d_hidden): - super().__init__() - self.lstm = nn.LSTM(d_model, d_hidden) - self._d_state = d_hidden - - @property - def d_state(self): - return self._d_state - - @property - def state_to_tensor(self): - return lambda state: state[0].view(state[0].size(1), -1) - - # ... other methods ... - - model = MyLSTMModel(d_model=64, d_hidden=32) - print(model.d_state) # Output: 32 - """ + """Return dimension of output of self.state_to_tensor.""" return None def default_state(self, *batch_shape, device=None): - """ - Create initial state for a batch of inputs. - - This method generates the default initial state for the sequence model. It is - particularly useful for models that maintain internal states, such as RNNs or - state space models. - - Args: - *batch_shape: Variable length argument list for the batch dimensions. - device (torch.device, optional): The device on which to create the state. - Defaults to None, which means the state will be created on the default device. - - Returns: - Any: The initial state for the model. The exact type and shape depend on the - specific implementation. Returns None by default if not implemented. - - Example: - class MyGRUModel(SequenceModule): - def __init__(self, d_model, d_hidden): - super().__init__() - self.gru = nn.GRU(d_model, d_hidden) - self.d_hidden = d_hidden - - def default_state(self, *batch_shape, device=None): - return torch.zeros(*batch_shape, self.d_hidden, device=device) - - # ... other methods ... - - model = MyGRUModel(d_model=64, d_hidden=32) - initial_state = model.default_state(16, device=torch.device('cuda')) - print(initial_state.shape) # torch.Size([16, 32]) - - Note: - Subclasses should override this method if they require a non-None initial state. - The method should be able to handle variable batch dimensions. - """ + """Create initial state for a batch of inputs.""" return None def step(self, x, state=None, **kwargs): - """ - Process one step of the input sequence. - - This method steps the model recurrently for a single step of the input sequence. - It is particularly useful for models that need to process sequences step-by-step, - such as in autoregressive generation or online inference scenarios. - - Args: - x (torch.Tensor): Input tensor for a single step. If the forward pass has - signature (B, L, H1) -> (B, L, H2), this method's input should generally - have shape (B, H1), where B is the batch size and H1 is the input dimension. - state (Any, optional): The current state of the model. Defaults to None. - **kwargs: Additional keyword arguments. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor for the single step, typically of shape (B, H2), - where H2 is the output dimension. - - Any: The updated state after processing this step. - - Raises: - NotImplementedError: If the subclass does not implement this method. - - Example: - class MyRNNModel(SequenceModule): - def __init__(self, d_model, d_hidden): - super().__init__() - self.rnn_cell = nn.RNNCell(d_model, d_hidden) - self.d_model = d_model - self.d_output = d_hidden - - def step(self, x, state=None): - if state is None: - state = torch.zeros(x.size(0), self.d_output, device=x.device) - output = self.rnn_cell(x, state) - return output, output - - # ... other methods ... - - model = MyRNNModel(d_model=64, d_hidden=32) - x = torch.randn(16, 64) # (batch_size, d_model) - output, new_state = model.step(x) - print(output.shape) # torch.Size([16, 32]) - - Note: - This method should be implemented by subclasses that support step-by-step - processing. The exact signature may vary depending on the specific model architecture. + """Step the model recurrently for one step of the input sequence. + + For example, this should correspond to unrolling an RNN for one step. + If the forward pass has signature (B, L, H1) -> (B, L, H2), + this method should generally have signature + (B, H1) -> (B, H2) with an optional recurrent state. """ raise NotImplementedError def TransposedModule(module): - """ - Decorator to transpose input and output of a SequenceModule. - - This decorator wraps a SequenceModule class to handle transposed input and output, - manage state, and absorb additional keyword arguments. It allows the module to - operate on either (B, L, H) or (B, H, L) shaped inputs, where B is batch size, - L is sequence length, and H is hidden dimension. - - Attributes: - transposed (bool): If True, the input and output tensors are transposed. - - Args: - module (type): The SequenceModule class to be wrapped. - - Returns: - type: A new class that wraps the input module with transposition functionality. - - Example: - @TransposedModule - class MySequenceModule(SequenceModule): - def __init__(self, d_model): - super().__init__() - self.d_model = d_model - self.d_output = d_model - - def forward(self, x, state=None): - # Process x - return x, state - - # Now MySequenceModule can handle both (B, L, H) and (B, H, L) inputs - model = MySequenceModule(d_model=64, transposed=True) - x = torch.randn(32, 64, 100) # (B, H, L) - output, _ = model(x) # output will be (32, 64, 100) - - Note: - The wrapped module's forward method should accept 'state' as an argument - and return both the processed tensor and the next state. + """Transpose module. + + Wrap a SequenceModule class to accept transposed parameter, + handle state, absorb kwargs """ # https://stackoverflow.com/a/65470430/1980685 @@ -462,42 +134,7 @@ def forward(self, x, state=None, **kwargs): @TransposedModule class SequenceIdentity(SequenceModule): - """ - A simple identity SequenceModule for testing purposes. - - This class implements a basic SequenceModule that passes the input through - unchanged. It is primarily used for testing and as a minimal example of - a SequenceModule implementation. - - The SequenceIdentity class is wrapped with the @TransposedModule decorator, - allowing it to handle both standard and transposed input formats. - - Attributes: - d_model (int): The input and output dimension of the model. - d_output (int): Alias for d_model, as this module doesn't change dimensions. - - Args: - d_model (int): The dimension of the input and output. - dropout (float, optional): Dropout rate (unused in this implementation). - Defaults to 0.0. - **kwargs: Additional keyword arguments (unused). - - Example: - model = SequenceIdentity(d_model=64) - x = torch.randn(32, 100, 64) # (batch_size, sequence_length, d_model) - output, _ = model(x) - assert torch.allclose(x, output) # True, as this is an identity module - - # With transposed input - model_transposed = SequenceIdentity(d_model=64, transposed=True) - x_transposed = torch.randn(32, 64, 100) # (batch_size, d_model, sequence_length) - output_transposed, _ = model_transposed(x_transposed) - assert torch.allclose(x_transposed, output_transposed) # True - - Note: - This class is wrapped with @TransposedModule, which allows it to handle - both (B, L, H) and (B, H, L) input formats based on the 'transposed' parameter. - """ + """Simple SequenceModule for testing purposes.""" def __init__(self, d_model, dropout=0.0, **kwargs): """Initialize SequenceModule. @@ -510,83 +147,13 @@ def __init__(self, d_model, dropout=0.0, **kwargs): self.d_output = d_model def forward(self, x, state=None): - """ - Perform the forward pass of the SequenceIdentity module. - - This method implements the identity operation, returning the input unchanged. - - Args: - x (torch.Tensor): Input tensor of shape (batch, length, d_model) if not transposed, - or (batch, d_model, length) if transposed. - state (Any, optional): Unused in this implementation. Defaults to None. - - Returns: - tuple: A tuple containing: - - torch.Tensor: The input tensor x, unchanged. - - None: As this module doesn't maintain state. - - Example: - model = SequenceIdentity(d_model=64) - x = torch.randn(32, 100, 64) # (batch_size, sequence_length, d_model) - output, _ = model(x) - assert torch.allclose(x, output) # True, as this is an identity operation - - Note: - If the module was initialized with transposed=True, the method expects - and returns tensors in the shape (batch, d_model, length). - """ + """Forward pass.""" return x, state def default_state(self, *batch_shape, device=None): - """ - Create initial state for a batch of inputs. - - This method returns None as the SequenceIdentity module does not maintain any state. - - Args: - *batch_shape: Variable length argument list for the batch dimensions (unused). - device (torch.device, optional): The device on which to create the state (unused). - Defaults to None. - - Returns: - None: As SequenceIdentity doesn't use any state. - - Example: - model = SequenceIdentity(d_model=64) - initial_state = model.default_state(32) # batch size of 32 - assert initial_state is None # True - - Note: - This method is implemented to conform to the SequenceModule interface, - but it doesn't perform any meaningful operation for SequenceIdentity. - """ + """Create initial state for a batch of inputs.""" return None def step(self, x, state=None, **kwargs): - """ - Process one step of the input sequence. - - This method implements the identity operation for a single step, returning the input unchanged. - - Args: - x (torch.Tensor): Input tensor for a single step, typically of shape (batch, d_model). - state (Any, optional): Unused in this implementation. Defaults to None. - **kwargs: Additional keyword arguments (unused). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The input tensor x, unchanged. - - None: As this module doesn't maintain state. - - Example: - model = SequenceIdentity(d_model=64) - x = torch.randn(32, 64) # (batch_size, d_model) - output, _ = model.step(x) - assert torch.allclose(x, output) # True, as this is an identity operation - - Note: - This method is implemented to conform to the SequenceModule interface. - For SequenceIdentity, it performs the same operation as the forward method, - but for a single step of the sequence. - """ + """Step the model recurrently for one step of the input sequence.""" return x, state diff --git a/espnet2/asr/state_spaces/block.py b/espnet2/asr/state_spaces/block.py index cf25155c17d..5f64aadbdad 100644 --- a/espnet2/asr/state_spaces/block.py +++ b/espnet2/asr/state_spaces/block.py @@ -26,47 +26,22 @@ class SequenceResidualBlock(SequenceModule): - """ - Residual block wrapper for black box layer. - - This class implements a generic (batch, length, d_input) -> (batch, length, d_input) transformation - with configurable normalization, pooling, and residual options. - - Attributes: - i_layer (int): Layer index, used by certain residual functions like Decay. - d_input (int): Input feature dimension. - layer (SequenceModule): Black box layer module. - prenorm (bool): If True, apply normalization before the layer; otherwise, after. - transposed (bool): If True, transpose inputs so each layer receives (batch, dim, length). - residual (ResidualFunction): Residual function module. - norm (Normalization): Normalization layer. - pool (PoolModule): Pooling layer. - drop (nn.Module): Dropout module. - drop_path (nn.Module): Stochastic depth module. + """Residual block wrapper for black box layer. + + The SequenceResidualBlock class implements a generic + (batch, length, d_input) -> (batch, length, d_input) transformation Args: - d_input (int): Input feature dimension. - i_layer (int, optional): Layer index. Defaults to None. - prenorm (bool, optional): Whether to apply normalization before the layer. Defaults to True. - dropout (float, optional): Dropout rate. Defaults to 0.0. - tie_dropout (bool, optional): Whether to tie dropout mask across sequence. Defaults to False. - transposed (bool, optional): Whether to transpose inputs. Defaults to False. - layer (dict, optional): Config for black box module. Defaults to None. - residual (dict, optional): Config for residual function. Defaults to None. - norm (str or dict, optional): Config for normalization layer. Defaults to None. - pool (dict, optional): Config for pooling layer. Defaults to None. - drop_path (float, optional): Drop ratio for stochastic depth. Defaults to 0.0. - - Example: - >>> block = SequenceResidualBlock(d_input=256, prenorm=True, dropout=0.1) - >>> x = torch.randn(32, 100, 256) # (batch, length, d_input) - >>> y, _ = block(x) - >>> y.shape - torch.Size([32, 100, 256]) - - Note: - The class supports various configurations for normalization, residual connections, - pooling, and regularization, making it highly flexible for different architectural designs. + d_input: Input feature dimension + i_layer: Layer index, only needs to be passed into certain residuals like Decay + dropout: Dropout for black box module + tie_dropout: Tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d + transposed: Transpose inputs so each layer receives (batch, dim, length) + layer: Config for black box module + residual: Config for residual function + norm: Config for normalization layer + pool: Config for pooling layer per stage + drop_path: Drop ratio for stochastic depth """ def __init__( @@ -136,94 +111,20 @@ def __init__( @property def d_output(self): - """ - Output dimension of the SequenceResidualBlock. - - Returns: - int: The output dimension. If a pooling layer is present, it returns the output - dimension of the pooling layer. Otherwise, it returns the residual dimension. - - Note: - This property dynamically computes the output dimension based on the block's - configuration, taking into account whether pooling is applied. - """ return self.pool.d_output if self.pool is not None else self.d_residual @property def d_state(self): - """ - State dimension of the underlying layer. - - Returns: - int: The state dimension of the black box layer. - - Note: - This property provides access to the state dimension of the internal layer, - which is useful for understanding the hidden state size in stateful models. - """ return self.layer.d_state @property def state_to_tensor(self): - """ - Method to convert the layer's state to a tensor. - - Returns: - callable: The state_to_tensor method of the underlying layer. - - Note: - This property provides access to the state_to_tensor method of the internal layer, - allowing for consistent state handling across the residual block wrapper. - """ return self.layer.state_to_tensor def default_state(self, *args, **kwargs): - """ - Get the default state for the underlying layer. - - This method delegates to the default_state method of the internal layer. - - Args: - *args: Variable length argument list to be passed to the layer's default_state method. - **kwargs: Arbitrary keyword arguments to be passed to the layer's default_state method. - - Returns: - The default state of the underlying layer. - - Note: - This method allows the SequenceResidualBlock to maintain the same interface - for state initialization as its internal layer. - """ return self.layer.default_state(*args, **kwargs) def forward(self, x, state=None, **kwargs): - """ - Forward pass of the SequenceResidualBlock. - - This method applies the full sequence of operations: normalization (if prenorm), - black box layer, residual connection, normalization (if postnorm), and pooling. - - Args: - x (torch.Tensor): Input tensor of shape (batch, length, d_input). - state (Any, optional): Initial state for the layer. Defaults to None. - **kwargs: Additional keyword arguments to be passed to the black box layer. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor after all operations. - - Any: Updated state of the black box layer. - - Example: - >>> block = SequenceResidualBlock(d_input=256) - >>> x = torch.randn(32, 100, 256) - >>> output, new_state = block(x) - >>> output.shape - torch.Size([32, 100, 256]) - - Note: - The order of operations and the application of each component (norm, residual, pool) - depends on the block's configuration. - """ y = x # Pre-norm @@ -248,35 +149,6 @@ def forward(self, x, state=None, **kwargs): return y, state def step(self, x, state, **kwargs): - """ - Perform a single step forward pass of the SequenceResidualBlock. - - This method is designed for sequential processing, applying the block's operations - on a single time step input. - - Args: - x (torch.Tensor): Input tensor for a single time step. - state (Any): Current state of the layer. - **kwargs: Additional keyword arguments to be passed to the black box layer's step method. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor after all operations for the current time step. - - Any: Updated state of the black box layer. - - Note: - This method follows a similar sequence of operations as the forward method, - but is adapted for step-by-step processing. It does not apply dropout or stochastic depth, - and the residual connection is applied without transposition. - - Example: - >>> block = SequenceResidualBlock(d_input=256) - >>> x = torch.randn(32, 256) # Single time step input - >>> state = block.default_state(batch_size=32) - >>> output, new_state = block.step(x, state) - >>> output.shape - torch.Size([32, 256]) - """ y = x # Pre-norm diff --git a/espnet2/asr/state_spaces/cauchy.py b/espnet2/asr/state_spaces/cauchy.py index 23a29f33f12..4d36ea92657 100644 --- a/espnet2/asr/state_spaces/cauchy.py +++ b/espnet2/asr/state_spaces/cauchy.py @@ -13,39 +13,13 @@ def cauchy_mult_torch( v: torch.Tensor, z: torch.Tensor, w: torch.Tensor, symmetric=True ) -> torch.Tensor: - """ - Compute the Cauchy kernel using PyTorch operations. - - This function calculates the Cauchy kernel for given input tensors. It supports - both symmetric and non-symmetric computations. - - Args: - v (torch.Tensor): Input tensor of shape (B, N), where B is the batch size - and N is the number of elements. - z (torch.Tensor): Input tensor of shape (L,), where L is the length. - w (torch.Tensor): Input tensor of shape (B, N), where B is the batch size - and N is the number of elements. - symmetric (bool, optional): If True, assumes v and w contain complex conjugate - pairs of the form [v_half, v_half.conj()] and [w_half, w_half.conj()]. - Defaults to True. - - Returns: - torch.Tensor: The computed Cauchy kernel. + """Compute Cauchy kernel. - Raises: - AssertionError: If symmetric is True and N is not even. - - Examples: - >>> v = torch.randn(2, 4) - >>> z = torch.randn(3) - >>> w = torch.randn(2, 4) - >>> result = cauchy_mult_torch(v, z, w) - >>> print(result.shape) - torch.Size([2, 3]) - - Note: - When symmetric is True, the function only uses the first half of v and w, - assuming they contain complex conjugate pairs. + v: (B, N) + z: (L) + w: (B, N) + symmetric: whether to assume that v and w contain complex conjugate pairs, of the + form [v_half, v_half.conj()] and [w_half, w_half.conj()] """ if not symmetric: return ( @@ -65,34 +39,6 @@ def cauchy_mult_torch( def cauchy_mult_keops(v, z, w): - """ - Compute the Cauchy kernel using KeOps LazyTensors. - - This function calculates the Cauchy kernel for given input tensors using - KeOps LazyTensors for efficient GPU computation. - - Args: - v (torch.Tensor): Input tensor of shape (b, N), where b is the batch size - and N is the number of elements. - z (torch.Tensor): Input tensor of shape (L,), where L is the length. - w (torch.Tensor): Input tensor of shape (b, N), where b is the batch size - and N is the number of elements. - - Returns: - torch.Tensor: The computed Cauchy kernel of shape (b, L). - - Note: - This function requires the PyKeOps library to be installed and - configured properly for GPU acceleration. - - Examples: - >>> v = torch.randn(2, 1000) - >>> z = torch.randn(500) - >>> w = torch.randn(2, 1000) - >>> result = cauchy_mult_keops(v, z, w) - >>> print(result.shape) - torch.Size([2, 500]) - """ from pykeops.torch import LazyTensor v_l = LazyTensor(rearrange(v, "b N -> b 1 N 1")) @@ -112,38 +58,7 @@ def _cauchy_mult(v, z, w, symmetric=True): def cauchy_mult(v, z, w, symmetric=True): - """ - Wrapper function for Cauchy multiplication that handles tensor shapes. - - This function wraps the CUDA-based Cauchy multiplication method, ensuring proper - tensor shapes and broadcasting before computation. - - Args: - v (torch.Tensor): Input tensor of shape (..., N), where N is the number of elements. - z (torch.Tensor): Input tensor of shape (L,), where L is the length. - w (torch.Tensor): Input tensor of shape (..., N), where N is the number of elements. - symmetric (bool, optional): If True, uses symmetric Cauchy multiplication. - Defaults to True. - - Returns: - torch.Tensor: The result of Cauchy multiplication with shape (..., L). - - Raises: - AssertionError: If z is not a 1-dimensional tensor after squeezing. - AssertionError: If the last dimensions of v and w do not match. - - Note: - This function broadcasts v and w tensors, making it flexible for various - input shapes. - - Examples: - >>> v = torch.randn(2, 3, 1000) - >>> z = torch.randn(500) - >>> w = torch.randn(2, 3, 1000) - >>> result = cauchy_mult(v, z, w) - >>> print(result.shape) - torch.Size([2, 3, 500]) - """ + """Wrap the cuda method to deal with shapes.""" v, w = torch.broadcast_tensors(v, w) shape = v.shape # z_shape = z.shape @@ -163,52 +78,8 @@ def cauchy_mult(v, z, w, symmetric=True): class CauchyMultiply(torch.autograd.Function): - """ - Custom autograd function for Cauchy multiplication. - - This class implements a custom autograd function for Cauchy multiplication, - providing both forward and backward passes. It utilizes CUDA operations - for efficient computation on GPU. - - Note: - This class is designed to be used with PyTorch's autograd system and - should not be instantiated directly. Instead, use it through the - `_cauchy_mult` function. - - Attributes: - Inherits attributes from torch.autograd.Function. - - Raises: - NotImplementedError: If input tensors are not CUDA tensors, or if the - input dimensions are not supported. - """ - @staticmethod def forward(ctx, v, z, w): - """ - Perform the forward pass of Cauchy multiplication. - - This method computes the Cauchy multiplication using CUDA operations. - - Args: - ctx (torch.autograd.function.FunctionCtx): Context object to save - information for backward computation. - v (torch.Tensor): Input tensor of shape (batch, N). - z (torch.Tensor): Input tensor of shape (L,). - w (torch.Tensor): Input tensor of shape (batch, N). - - Returns: - torch.Tensor: Result of Cauchy multiplication. - - Raises: - NotImplementedError: If N is not in the supported values list. - NotImplementedError: If L is not a multiple of 32. - NotImplementedError: If input tensors are not CUDA tensors. - - Note: - Currently only supports N values of 64 (2^6) and L values that are - multiples of 32. - """ batch, N = v.shape # supported_N_values = [1 << log_n for log_n in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] supported_N_values = [1 << log_n for log_n in [6]] @@ -224,79 +95,14 @@ def forward(ctx, v, z, w): @staticmethod def backward(ctx, dout): - """ - Perform the backward pass of Cauchy multiplication. - - This method computes the gradients with respect to inputs v and w. - - Args: - ctx (torch.autograd.function.FunctionCtx): Context object containing - saved tensors from the forward pass. - dout (torch.Tensor): Gradient of the loss with respect to the output - of the forward pass. - - Returns: - tuple: A tuple containing: - - dv (torch.Tensor): Gradient with respect to input v. - - None: Placeholder for gradient with respect to z (not computed). - - dw (torch.Tensor): Gradient with respect to input w. - - Note: - The gradient with respect to z is not computed and returned as None. - """ v, z, w = ctx.saved_tensors dv, dw = cauchy_mult_bwd(v, z, w, dout) return dv, None, dw class CauchyMultiplySymmetric(torch.autograd.Function): - """ - Custom autograd function for symmetric Cauchy multiplication. - - This class implements a custom autograd function for symmetric Cauchy - multiplication, providing both forward and backward passes. It utilizes - CUDA operations for efficient computation on GPU, assuming symmetry in - the input tensors. - - Note: - This class is designed to be used with PyTorch's autograd system and - should not be instantiated directly. Instead, use it through the - `_cauchy_mult` function with the symmetric option set to True. - - Attributes: - Inherits attributes from torch.autograd.Function. - - Raises: - NotImplementedError: If input tensors are not CUDA tensors, or if the - input dimensions are not supported. - """ - @staticmethod def forward(ctx, v, z, w): - """ - Perform the forward pass of symmetric Cauchy multiplication. - - This method computes the symmetric Cauchy multiplication using CUDA operations. - - Args: - ctx (torch.autograd.function.FunctionCtx): Context object to save - information for backward computation. - v (torch.Tensor): Input tensor of shape (batch, N). - z (torch.Tensor): Input tensor of shape (L,). - w (torch.Tensor): Input tensor of shape (batch, N). - - Returns: - torch.Tensor: Result of symmetric Cauchy multiplication. - - Raises: - NotImplementedError: If N is not in the supported values list. - NotImplementedError: If L exceeds the maximum supported value. - NotImplementedError: If input tensors are not CUDA tensors. - - Note: - Supports N values that are powers of 2 from 2 to 1024. - The maximum supported L value is 32 * 1024 * 64 * 1024. - """ batch, N = v.shape supported_N_values = [1 << log_n for log_n in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] L = z.shape[-1] @@ -312,29 +118,6 @@ def forward(ctx, v, z, w): @staticmethod def backward(ctx, dout): - """ - Perform the backward pass of symmetric Cauchy multiplication. - - This method computes the gradients with respect to inputs v and w for the - symmetric case. - - Args: - ctx (torch.autograd.function.FunctionCtx): Context object containing - saved tensors from the forward pass. - dout (torch.Tensor): Gradient of the loss with respect to the output - of the forward pass. - - Returns: - tuple: A tuple containing: - - dv (torch.Tensor): Gradient with respect to input v. - - None: Placeholder for gradient with respect to z (not computed). - - dw (torch.Tensor): Gradient with respect to input w. - - Note: - The gradient with respect to z is not computed and returned as None. - This method assumes symmetry in the input tensors and uses specialized - CUDA operations for symmetric Cauchy multiplication. - """ v, z, w = ctx.saved_tensors dv, dw = cauchy_mult_sym_bwd(v, z, w, dout) return dv, None, dw diff --git a/espnet2/asr/state_spaces/components.py b/espnet2/asr/state_spaces/components.py index f20236fc688..b6062be2b18 100644 --- a/espnet2/asr/state_spaces/components.py +++ b/espnet2/asr/state_spaces/components.py @@ -11,35 +11,23 @@ def stochastic_depth(input: torch.tensor, p: float, mode: str, training: bool = True): - """ - Apply stochastic depth to the input tensor. + """Apply stochastic depth. - Implements the Stochastic Depth technique from "Deep Networks with Stochastic Depth" - (https://arxiv.org/abs/1603.09382) used for randomly dropping residual branches - of residual architectures. + Implements the Stochastic Depth from `"Deep Networks with Stochastic Depth" + `_ used for randomly dropping residual + branches of residual architectures. Args: - input (torch.tensor): The input tensor of arbitrary dimensions with the first - dimension being its batch size. - p (float): Probability of the input to be zeroed. - mode (str): Either "batch" or "row". "batch" randomly zeroes the entire input, - "row" zeroes randomly selected rows from the batch. - training (bool, optional): Apply stochastic depth if True. Defaults to True. + input (Tensor[N, ...]): The input tensor or arbitrary dimensions with the first + one being its batch i.e. a batch with ``N`` rows. + p (float): probability of the input to be zeroed. + mode (str): ``"batch"`` or ``"row"``. + ``"batch"`` randomly zeroes the entire input, ``"row"`` zeroes + randomly selected rows from the batch. + training: apply stochastic depth if is ``True``. Default: ``True`` Returns: - torch.tensor: The randomly zeroed tensor. - - Raises: - ValueError: If p is not between 0 and 1, or if mode is not "batch" or "row". - - Examples: - >>> input_tensor = torch.randn(32, 64, 224, 224) - >>> output = stochastic_depth(input_tensor, p=0.2, mode="batch") - >>> print(output.shape) - torch.Size([32, 64, 224, 224]) - - Note: - When training is False or p is 0, the function returns the input tensor unchanged. + Tensor[N, ...]: The randomly zeroed tensor. """ if p < 0.0 or p > 1.0: raise ValueError( @@ -63,21 +51,9 @@ def stochastic_depth(input: torch.tensor, p: float, mode: str, training: bool = class StochasticDepth(nn.Module): - """ - A module that applies stochastic depth to its input. + """Stochastic depth module. - This module implements the Stochastic Depth technique from the paper - "Deep Networks with Stochastic Depth" (https://arxiv.org/abs/1603.09382). - It randomly drops entire layers or parts of the input during training, - which can help in regularizing deep networks. - - Attributes: - p (float): The probability of dropping the input. - mode (str): The mode of dropping, either "batch" or "row". - - Note: - This implementation is a custom version and may need to be upgraded - to use the official `torchvision.ops.StochasticDepth` in future versions. + See :func:`stochastic_depth`. """ def __init__(self, p: float, mode: str) -> None: @@ -88,22 +64,6 @@ def __init__(self, p: float, mode: str) -> None: self.mode = mode def forward(self, input): - """ - Apply stochastic depth to the input tensor. - - This method applies the stochastic depth technique to the input tensor - based on the probability and mode specified during initialization. - - Args: - input (torch.Tensor): The input tensor to apply stochastic depth to. - - Returns: - torch.Tensor: The output tensor after applying stochastic depth. - - Note: - The behavior of this method depends on whether the module is in - training mode or not, as well as the specified probability and mode. - """ return stochastic_depth(input, self.p, self.mode, self.training) def __repr__(self) -> str: @@ -115,23 +75,6 @@ def __repr__(self) -> str: class DropoutNd(nn.Module): - """ - A module that applies n-dimensional dropout to its input. - - This module extends the concept of dropout to n-dimensional tensors, - allowing for more flexible dropout patterns in complex neural network - architectures. - - Attributes: - p (float): The probability of an element to be zeroed. Must be between 0 and 1. - tie (bool): If True, ties the dropout mask across sequence lengths. - transposed (bool): If True, assumes the input tensor is in a transposed format. - - Note: - This implementation uses a custom dropout mechanism and may behave - differently from PyTorch's built-in dropout functions. - """ - def __init__(self, p: float = 0.5, tie=True, transposed=True): """Initialize dropout module. @@ -148,28 +91,9 @@ def __init__(self, p: float = 0.5, tie=True, transposed=True): self.binomial = torch.distributions.binomial.Binomial(probs=1 - self.p) def forward(self, X): - """ - Apply n-dimensional dropout to the input tensor. - - This method applies dropout to the input tensor based on the probability - and configuration specified during initialization. + """Forward pass. - Args: - X (torch.Tensor): The input tensor of shape (batch, dim, lengths...). - - Returns: - torch.Tensor: The output tensor after applying dropout. - - Note: - The dropout is only applied during training mode. In evaluation mode, - the input tensor is returned unchanged. - - Examples: - >>> dropout = DropoutNd(p=0.5, tie=True, transposed=False) - >>> input = torch.randn(32, 64, 10, 10) - >>> output = dropout(input) - >>> print(output.shape) - torch.Size([32, 64, 10, 10]) + X: (batch, dim, lengths...) """ if self.training: if not self.transposed: @@ -187,42 +111,6 @@ def forward(self, X): def Activation(activation=None, size=None, dim=-1): - """ - Create and return an activation layer based on the specified activation type. - - This function provides a convenient way to create various activation layers - commonly used in neural networks. - - Args: - activation (str, optional): The type of activation to use. Supported values are: - None, "id", "identity", "linear", "tanh", "relu", "gelu", "swish", "silu", - "glu", "sigmoid", "sqrelu", "ln". Defaults to None. - size (int, optional): The size parameter for certain activation types. - Only used for specific activations. Defaults to None. - dim (int, optional): The dimension along which to apply the activation for - dimension-specific activations like GLU. Defaults to -1. - - Returns: - nn.Module: An instance of the specified activation layer. - - Raises: - NotImplementedError: If the specified activation is not implemented. - - Examples: - >>> relu_activation = Activation("relu") - >>> output = relu_activation(torch.randn(10, 20)) - >>> print(type(output)) - - - >>> glu_activation = Activation("glu", dim=1) - >>> output = glu_activation(torch.randn(10, 20, 30)) - >>> print(output.shape) - torch.Size([10, 10, 30]) - - Note: - Some activation types (e.g., "ln") may require additional parameters - that are not directly exposed in this function's interface. - """ if activation in [None, "id", "identity", "linear"]: return nn.Identity() elif activation == "tanh": @@ -248,39 +136,6 @@ def Activation(activation=None, size=None, dim=-1): def get_initializer(name, activation=None): - """ - Get an initialization function based on the specified name and activation. - - This function returns an initializer function that can be used to initialize - the weights of neural network layers. - - Args: - name (str): The name of the initializer. Supported values are: - "uniform", "normal", "xavier", "zero", "one". - activation (str, optional): The activation function used in the layer. - This is used to determine the appropriate gain for certain initializers. - Supported values are: None, "id", "identity", "linear", "modrelu", "relu", - "tanh", "sigmoid", "gelu", "swish", "silu". Defaults to None. - - Returns: - callable: An initialization function that can be applied to tensor weights. - - Raises: - NotImplementedError: If the specified name or activation is not supported. - - Examples: - >>> init_func = get_initializer("uniform", activation="relu") - >>> linear = nn.Linear(10, 20) - >>> init_func(linear.weight) - - >>> xavier_init = get_initializer("xavier") - >>> conv = nn.Conv2d(3, 64, 3) - >>> xavier_init(conv.weight) - - Note: - The returned initializer functions are partial functions from PyTorch's - initialization methods, with pre-set parameters based on the input arguments. - """ if activation in [None, "id", "identity", "linear", "modrelu"]: nonlinearity = "linear" elif activation in ["relu", "tanh", "sigmoid"]: @@ -322,44 +177,7 @@ def LinearActivation( weight_norm=False, **kwargs, ): - """ - Create a linear module with optional initialization and activation. - - This function constructs a linear layer with various options for initialization, - activation, and normalization. - - Args: - d_input (int): Dimension of the input features. - d_output (int): Dimension of the output features. - bias (bool, optional): If True, adds a learnable bias to the layer. Defaults to True. - zero_bias_init (bool, optional): If True, initializes the bias to zero. Defaults to False. - transposed (bool, optional): If True, uses TransposedLinear instead of nn.Linear. Defaults to False. - initializer (str, optional): Initialization method for the weights. Defaults to None. - activation (str, optional): Activation function to use. Defaults to None. - activate (bool, optional): If True, applies the activation as part of this module. Defaults to False. - weight_norm (bool, optional): If True, applies weight normalization. Defaults to False. - **kwargs: Additional keyword arguments passed to the linear layer. - - Returns: - nn.Module: A linear module, possibly followed by an activation function. - - Examples: - >>> layer = LinearActivation(100, 200, activation='relu', activate=True) - >>> input = torch.randn(32, 100) - >>> output = layer(input) - >>> print(output.shape) - torch.Size([32, 200]) - - >>> transposed_layer = LinearActivation(50, 75, transposed=True, weight_norm=True) - >>> input = torch.randn(16, 50, 10) - >>> output = transposed_layer(input) - >>> print(output.shape) - torch.Size([16, 75, 10]) - - Note: - If activation is set to 'glu', the output dimension is doubled internally - before applying the activation. - """ + """Return a linear module, initialization, and activation.""" # Construct core module # linear_cls = partial(nn.Conv1d, kernel_size=1) if transposed else nn.Linear linear_cls = TransposedLinear if transposed else nn.Linear @@ -386,65 +204,15 @@ def LinearActivation( class SquaredReLU(nn.Module): - """ - A module that applies a squared ReLU activation function. - - This activation function first applies a standard ReLU (Rectified Linear Unit) - activation, then squares the result. It can be used as a non-linear activation - in neural networks, potentially capturing more complex patterns than standard ReLU. - - The function is defined as: - f(x) = (max(0, x))^2 - - Note: - This activation function may lead to different gradient behavior compared - to standard ReLU, potentially affecting the training dynamics of the network. - """ - def forward(self, x): - """ - Apply the squared ReLU activation function to the input. - - This method applies the ReLU function to the input and then squares the result. - - Args: - x (torch.Tensor): The input tensor. - - Returns: - torch.Tensor: The output tensor after applying the squared ReLU activation. - - Examples: - >>> squared_relu = SquaredReLU() - >>> input = torch.randn(10) - >>> output = squared_relu(input) - >>> print(output.shape) - torch.Size([10]) - - Note: - For negative input values, the output will be zero. For positive input values, - the output will be the square of the input. - """ return F.relu(x) ** 2 class TransposedLinear(nn.Module): - """ - A linear module that operates on the second-to-last dimension of the input. - - This module implements a linear transformation similar to nn.Linear, but applies - the transformation on the second-to-last dimension of the input tensor. It is - designed to handle input tensors of shape (B, D, L), where B is the batch size, - D is the input dimension, and L represents one or more additional dimensions. - - Attributes: - weight (nn.Parameter): The learnable weights of the module of shape (d_output, d_input). - bias (nn.Parameter or float): The learnable bias of the module of shape (d_output,). - If bias is False, then the layer does not use a bias. - - Note: - This module can be particularly useful in scenarios where the standard linear - layer's dimension handling is not suitable, such as in certain types of - sequence processing or when working with multi-dimensional data. + """Transposed linear module. + + Linear module on the second-to-last dimension + Assumes shape (B, D, L), where L can be 1 or more axis """ def __init__(self, d_input, d_output, bias=True): @@ -464,30 +232,6 @@ def __init__(self, d_input, d_output, bias=True): self.bias = 0.0 def forward(self, x): - """ - Apply a linear transformation to the input tensor. - - This method performs the linear transformation on the second-to-last dimension - of the input tensor. - - Args: - x (torch.Tensor): The input tensor of shape (B, D, ...), where B is the batch size, - D is the input dimension, and ... represents any number of additional dimensions. - - Returns: - torch.Tensor: The output tensor of shape (B, V, ...), where V is the output dimension. - - Examples: - >>> transposed_linear = TransposedLinear(64, 128) - >>> input = torch.randn(32, 64, 10, 10) - >>> output = transposed_linear(input) - >>> print(output.shape) - torch.Size([32, 128, 10, 10]) - - Note: - The transformation is applied using einsum for efficient computation, - especially for inputs with multiple trailing dimensions. - """ num_axis = len(x.shape[2:]) # num_axis in L, for broadcasting bias y = contract("b u ..., v u -> b v ...", x, self.weight) + self.bias.view( -1, *[1] * num_axis @@ -496,23 +240,13 @@ def forward(self, x): class TransposedLN(nn.Module): - """ - A transposed Layer Normalization module. - - This module applies Layer Normalization over the second dimension of the input tensor. - It is designed to handle input tensors of shape (B, D, L), where B is the batch size, - D is the feature dimension to be normalized, and L represents one or more additional dimensions. - - Attributes: - scalar (bool): If True, uses learnable scalar parameters for affine transformation. - If False, uses a full LayerNorm. - m (nn.Parameter): Learnable shift parameter when scalar is True. - s (nn.Parameter): Learnable scale parameter when scalar is True. - ln (nn.LayerNorm): LayerNorm module used when scalar is False. - - Note: - This implementation may be slower than a dedicated CUDA/Triton implementation. - Future optimizations could provide substantial end-to-end speedup. + """Transposed LayerNorm module. + + LayerNorm module over second dimension + Assumes shape (B, D, L), where L can be 1 or more axis + + This is slow and a dedicated CUDA/Triton implementation + shuld provide substantial end-to-end speedup """ def __init__(self, d, scalar=True): @@ -527,30 +261,6 @@ def __init__(self, d, scalar=True): self.ln = nn.LayerNorm(d) def forward(self, x): - """ - Apply transposed Layer Normalization to the input tensor. - - This method normalizes the input tensor along the second dimension (D). - - Args: - x (torch.Tensor): The input tensor of shape (B, D, ...), where B is the batch size, - D is the feature dimension to be normalized, and ... represents any number of - additional dimensions. - - Returns: - torch.Tensor: The normalized output tensor of the same shape as the input. - - Examples: - >>> transposed_ln = TransposedLN(64, scalar=True) - >>> input = torch.randn(32, 64, 10, 10) - >>> output = transposed_ln(input) - >>> print(output.shape) - torch.Size([32, 64, 10, 10]) - - Note: - When scalar is True, it uses learnable scalar parameters for affine transformation. - When scalar is False, it applies full LayerNorm by rearranging the tensor dimensions. - """ if self.scalar: # calc. stats over D dim / channels s, m = torch.std_mean(x, dim=1, unbiased=False, keepdim=True) @@ -564,25 +274,6 @@ def forward(self, x): class Normalization(nn.Module): - """ - A flexible normalization module supporting various normalization techniques. - - This module provides a unified interface for different types of normalization, - including Layer Normalization, Instance Normalization, Batch Normalization, - and Group Normalization. It can handle both standard and transposed input formats. - - Attributes: - transposed (bool): If True, assumes the length dimension is -1 or -2. - _name_ (str): The type of normalization to use. Options are "layer", "instance", - "batch", "group", or "none". - channel (bool): If True, normalization is applied over the channel dimension. - norm (nn.Module): The actual normalization module based on the specified type. - - Note: - The behavior and performance of this module can vary significantly based on - the chosen normalization type and the structure of the input data. - """ - def __init__( self, d, @@ -622,30 +313,6 @@ def __init__( raise NotImplementedError def forward(self, x): - """ - Apply the specified normalization to the input tensor. - - This method reshapes the input tensor if necessary, applies the chosen - normalization technique, and then restores the original shape. - - Args: - x (torch.Tensor): The input tensor to be normalized. The shape depends on - the 'transposed' attribute and the specific normalization technique. - - Returns: - torch.Tensor: The normalized tensor, maintaining the same shape as the input. - - Examples: - >>> norm = Normalization(64, transposed=False, _name_="layer") - >>> input = torch.randn(32, 10, 64) - >>> output = norm(input) - >>> print(output.shape) - torch.Size([32, 10, 64]) - - Note: - The method handles higher dimensional inputs by reshaping them before - normalization and then restoring the original shape afterwards. - """ # Handle higher dimension logic shape = x.shape if self.transposed: @@ -667,36 +334,6 @@ def forward(self, x): return x def step(self, x, **kwargs): - """ - Apply normalization to a single step input. - - This method is designed for use in scenarios where normalization needs to be - applied to a single time step or element, such as in recurrent neural networks - or sequential processing. - - Args: - x (torch.Tensor): The input tensor representing a single step or element. - **kwargs: Additional keyword arguments that might be required for specific - normalization types. - - Returns: - torch.Tensor: The normalized tensor for the single step. - - Raises: - AssertionError: If the normalization type is not one of "layer", "instance", - "batch", "group", or "none". - - Examples: - >>> norm = Normalization(64, transposed=True, _name_="layer") - >>> step_input = torch.randn(32, 64) - >>> normalized_step = norm.step(step_input) - >>> print(normalized_step.shape) - torch.Size([32, 64]) - - Note: - This method adds an extra dimension to the input if 'transposed' is True, - applies normalization, and then squeezes the added dimension. - """ assert self._name_ in ["layer", "instance", "batch", "group", "none"] if self.transposed: x = x.unsqueeze(-1) @@ -707,22 +344,6 @@ def step(self, x, **kwargs): class TSNormalization(nn.Module): - """ - A module for time series normalization. - - This class implements normalization techniques specifically designed for time series data. - It supports different methods of normalization based on the temporal characteristics of the input. - - Attributes: - method (str): The normalization method to use. Supported values are "mean" and "last". - horizon (int): The number of time steps to consider for normalization. - - Note: - This normalization is particularly useful in time series forecasting tasks, - where different parts of the time series may need to be treated differently - for effective normalization. - """ - def __init__(self, method, horizon): super().__init__() @@ -730,30 +351,6 @@ def __init__(self, method, horizon): self.horizon = horizon def forward(self, x): - """ - Apply time series normalization to the input tensor. - - This method normalizes the input time series data based on the specified method and horizon. - - Args: - x (torch.Tensor): The input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the feature dimension. - - Returns: - torch.Tensor: The normalized tensor of the same shape as the input. - - Examples: - >>> ts_norm = TSNormalization(method="mean", horizon=24) - >>> input = torch.randn(32, 100, 64) # 32 batches, 100 time steps, 64 features - >>> output = ts_norm(input) - >>> print(output.shape) - torch.Size([32, 100, 64]) - - Note: - - For the "mean" method, it uses the mean of absolute values up to the horizon for normalization. - - For the "last" method, it uses the last value before the horizon for normalization. - - If the method is neither "mean" nor "last", the input is returned unchanged. - """ # x must be BLD if self.method == "mean": self.scale = x.abs()[:, : -self.horizon].mean(dim=1)[:, None, :] @@ -765,22 +362,6 @@ def forward(self, x): class TSInverseNormalization(nn.Module): - """ - A module for inverting time series normalization. - - This class implements the inverse operation of TSNormalization, allowing - the restoration of normalized time series data to its original scale. - - Attributes: - method (str): The normalization method that was used. Supported values are "mean" and "last". - normalizer (TSNormalization): The TSNormalization instance that was used for the initial normalization. - - Note: - This module is typically used in conjunction with TSNormalization to revert - normalized predictions or processed data back to their original scale in - time series analysis and forecasting tasks. - """ - def __init__(self, method, normalizer): super().__init__() @@ -788,57 +369,12 @@ def __init__(self, method, normalizer): self.normalizer = normalizer def forward(self, x): - """ - Apply inverse time series normalization to the input tensor. - - This method reverses the normalization applied by TSNormalization, restoring - the data to its original scale. - - Args: - x (torch.Tensor): The normalized input tensor of shape (B, L, D), where B is the batch size, - L is the sequence length, and D is the feature dimension. - - Returns: - torch.Tensor: The denormalized tensor of the same shape as the input. - - Examples: - >>> ts_norm = TSNormalization(method="mean", horizon=24) - >>> ts_inverse_norm = TSInverseNormalization(method="mean", normalizer=ts_norm) - >>> original_input = torch.randn(32, 100, 64) - >>> normalized = ts_norm(original_input) - >>> denormalized = ts_inverse_norm(normalized) - >>> print(torch.allclose(original_input, denormalized)) - True - - Note: - - For the "mean" and "last" methods, it multiplies the input by the scale stored in the normalizer. - - If the method is neither "mean" nor "last", the input is returned unchanged. - """ if self.method == "mean" or self.method == "last": return x * self.normalizer.scale return x class ReversibleInstanceNorm1dInput(nn.Module): - """ - A reversible instance normalization module for 1D input. - - This module applies instance normalization to the input in a way that allows - for reversing the normalization process later. It is designed to work with - 1D data, such as time series or sequential data. - - Attributes: - transposed (bool): If True, expects input in BDL format, otherwise in BLD format. - norm (nn.InstanceNorm1d): The instance normalization layer. - s (torch.Tensor): Standard deviation of the input, computed during forward pass. - m (torch.Tensor): Mean of the input, computed during forward pass. - - Note: - This module is typically used in pairs with ReversibleInstanceNorm1dOutput - to allow for normalization that can be undone, which can be useful in - certain types of neural network architectures or processing pipelines. - """ - def __init__(self, d, transposed=False): super().__init__() # BLD if transpoed is False, otherwise BDL @@ -846,32 +382,6 @@ def __init__(self, d, transposed=False): self.norm = nn.InstanceNorm1d(d, affine=True, track_running_stats=False) def forward(self, x): - """ - Apply reversible instance normalization to the input tensor. - - This method normalizes the input tensor and stores the statistics needed for - reversing the normalization later. - - Args: - x (torch.Tensor): The input tensor. If transposed is False, expected shape is (B, L, D), - otherwise (B, D, L), where B is batch size, L is sequence length, and D is feature dimension. - - Returns: - torch.Tensor: The normalized tensor with the same shape as the input. - - Examples: - >>> rev_norm = ReversibleInstanceNorm1dInput(64, transposed=False) - >>> input = torch.randn(32, 100, 64) # 32 batches, 100 time steps, 64 features - >>> output = rev_norm(input) - >>> print(output.shape) - torch.Size([32, 100, 64]) - - Note: - - The method computes and stores the mean and standard deviation of the input. - - A small epsilon (1e-4) is added to the standard deviation to avoid division by zero. - - The normalization is applied along the last dimension for non-transposed input, - and along the second-to-last dimension for transposed input. - """ # Means, stds if not self.transposed: x = x.transpose(-1, -2) @@ -888,23 +398,6 @@ def forward(self, x): class ReversibleInstanceNorm1dOutput(nn.Module): - """ - A module for reversing the instance normalization applied by ReversibleInstanceNorm1dInput. - - This module is designed to work in conjunction with ReversibleInstanceNorm1dInput - to undo the normalization process, restoring the data to its original scale and distribution. - - Attributes: - transposed (bool): If True, expects input in BDL format, otherwise in BLD format. - weight (nn.Parameter): The weight parameter from the input normalization module. - bias (nn.Parameter): The bias parameter from the input normalization module. - norm_input (ReversibleInstanceNorm1dInput): The input normalization module used for the forward pass. - - Note: - This module should be used with tensors that have been normalized by a corresponding - ReversibleInstanceNorm1dInput instance to ensure correct denormalization. - """ - def __init__(self, norm_input): super().__init__() self.transposed = norm_input.transposed @@ -913,32 +406,6 @@ def __init__(self, norm_input): self.norm_input = norm_input def forward(self, x): - """ - Reverse the instance normalization applied to the input tensor. - - This method denormalizes the input tensor using the statistics stored in the - corresponding ReversibleInstanceNorm1dInput module. - - Args: - x (torch.Tensor): The normalized input tensor. If transposed is False, expected shape is (B, L, D), - otherwise (B, D, L), where B is batch size, L is sequence length, and D is feature dimension. - - Returns: - torch.Tensor: The denormalized tensor with the same shape as the input. - - Examples: - >>> rev_norm_input = ReversibleInstanceNorm1dInput(64, transposed=False) - >>> rev_norm_output = ReversibleInstanceNorm1dOutput(rev_norm_input) - >>> original = torch.randn(32, 100, 64) - >>> normalized = rev_norm_input(original) - >>> denormalized = rev_norm_output(normalized) - >>> print(torch.allclose(original, denormalized, atol=1e-6)) - True - - Note: - - The denormalization process uses the mean and standard deviation stored in the norm_input attribute. - - If the input was transposed during normalization, it will be transposed again during denormalization. - """ if not self.transposed: x = x.transpose(-1, -2) diff --git a/espnet2/asr/state_spaces/ff.py b/espnet2/asr/state_spaces/ff.py index 93615d2cc7f..67ac605eda6 100644 --- a/espnet2/asr/state_spaces/ff.py +++ b/espnet2/asr/state_spaces/ff.py @@ -11,39 +11,6 @@ class FF(SequenceModule): - """ - Feed-Forward Network (FFN) block in the style of Transformers. - - This class implements a configurable Feed-Forward Network block, commonly used in - Transformer architectures. It consists of two linear layers with an activation - function and optional dropout in between. - - Attributes: - d_output (int): The output dimension of the FFN block. - transposed (bool): If True, the input is expected to be in transposed form. - ff (nn.Sequential): The sequential layers that make up the FFN block. - - Args: - d_input (int): The input dimension. - expand (int, optional): The expansion factor for the hidden dimension. Defaults to 2. - d_output (int, optional): The output dimension. If None, it's set to d_input. Defaults to None. - transposed (bool, optional): Whether the input is transposed. Defaults to False. - activation (str, optional): The activation function to use. Defaults to "gelu". - initializer (callable, optional): The initializer for the linear layers. Defaults to None. - dropout (float, optional): The dropout rate. Defaults to 0.0. - tie_dropout (bool, optional): If True, uses the same dropout mask for all elements. Defaults to False. - - Example: - >>> ffn = FF(d_input=512, expand=4, d_output=512, dropout=0.1) - >>> x = torch.randn(32, 100, 512) # (batch_size, seq_len, d_input) - >>> output, _ = ffn(x) - >>> print(output.shape) - torch.Size([32, 100, 512]) - - Note: - This implementation is derived from the state-spaces repository by Hazy Research. - """ - def __init__( self, d_input, @@ -92,64 +59,9 @@ def __init__( ) def forward(self, x, *args, **kwargs): - """ - Forward pass of the Feed-Forward Network. - - This method applies the feed-forward transformation to the input tensor. - - Args: - x (torch.Tensor): The input tensor. Shape depends on the 'transposed' attribute: - - If transposed=False: [batch_size, seq_len, d_input] - - If transposed=True: [batch_size, d_input, seq_len] - *args: Variable length argument list. Not used in this method but included for compatibility. - **kwargs: Arbitrary keyword arguments. Not used in this method but included for compatibility. - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor after applying the feed-forward transformation. - Shape will be the same as the input tensor, with the last dimension changed to d_output. - - None: Placeholder for consistency with other modules that might return a state. - - Example: - >>> ffn = FF(d_input=512, d_output=512) - >>> x = torch.randn(32, 100, 512) # (batch_size, seq_len, d_input) - >>> output, _ = ffn.forward(x) - >>> print(output.shape) - torch.Size([32, 100, 512]) - """ return self.ff(x), None def step(self, x, state, **kwargs): - """ - Perform a single step of the Feed-Forward Network. - - This method is designed for sequential processing, applying the feed-forward - transformation to a single time step of the input. - - Args: - x (torch.Tensor): The input tensor for a single time step. - - If transposed=False: Shape is [batch_size, d_input] - - If transposed=True: Shape is [batch_size, d_input, 1] - state: Unused parameter, included for compatibility with other sequential modules. - **kwargs: Additional keyword arguments. Not used in this method but included for compatibility. - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor after applying the feed-forward transformation. - Shape will be [batch_size, d_output] if not transposed, or [batch_size, d_output, 1] if transposed. - - Any: The unchanged state parameter, returned for consistency with other sequential modules. - - Example: - >>> ffn = FF(d_input=512, d_output=512, transposed=False) - >>> x = torch.randn(32, 512) # (batch_size, d_input) - >>> output, _ = ffn.step(x, None) - >>> print(output.shape) - torch.Size([32, 512]) - - Note: - The behavior of this method depends on the 'transposed' attribute of the FF class. - When transposed=True, the input is unsqueezed before processing and squeezed afterwards. - """ # x: [batch, d_input] if self.transposed: # expects: [batch, d_input, seq_len] diff --git a/espnet2/asr/state_spaces/model.py b/espnet2/asr/state_spaces/model.py index 1edf74fae0b..a962b11a45a 100644 --- a/espnet2/asr/state_spaces/model.py +++ b/espnet2/asr/state_spaces/model.py @@ -13,42 +13,26 @@ class SequenceModel(SequenceModule): - """ - Isotropic deep sequence model backbone, in the style of ResNets / Transformers. + """Isotropic deep sequence model backbone, in the style of ResNets / Transformers. - This class implements a generic (batch, length, d_input) -> (batch, length, d_output) transformation. + The SequenceModel class implements a generic + (batch, length, d_input) -> (batch, length, d_output) transformation Args: - d_model (int): Dimension of the model. Used to resize input (useful for deep models with residuals). - n_layers (int, optional): Number of layers. Defaults to 1. - transposed (bool, optional): If True, transpose inputs so each layer receives (batch, dim, length). Defaults to False. - dropout (float, optional): Dropout parameter applied on every residual and every layer. Defaults to 0.0. - tie_dropout (bool, optional): If True, tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d. Defaults to False. - prenorm (bool, optional): If True, use pre-norm instead of post-norm. Defaults to True. - n_repeat (int, optional): Number of times each layer is repeated per stage before applying pooling. Defaults to 1. - layer (dict or list, optional): Layer configuration. Must be specified. - residual (dict, optional): Residual configuration. - norm (dict or str, optional): Normalization configuration (e.g., layer vs batch). - pool (dict, optional): Configuration for pooling layer per stage. - track_norms (bool, optional): If True, log norms of each layer output. Defaults to True. - dropinp (float, optional): Input dropout rate. Defaults to 0.0. - drop_path (float, optional): Stochastic depth rate for each residual path. Defaults to 0.0. - - Attributes: - d_output (int): Output dimension of the model. - layers (nn.ModuleList): List of SequenceResidualBlock modules. - norm (nn.Module): Normalization layer (if prenorm is True) or nn.Identity. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4, dropout=0.1) - >>> inputs = torch.randn(32, 100, 256) # (batch, length, d_input) - >>> outputs, _ = model(inputs) - >>> outputs.shape - torch.Size([32, 100, 256]) - - Note: - The model can handle both transposed and non-transposed inputs, depending on the 'transposed' parameter. - It also supports various configurations for normalization, residual connections, and pooling. + d_model: Resize input (useful for deep models with residuals) + n_layers: Number of layers + transposed: Transpose inputs so each layer receives (batch, dim, length) + dropout: Dropout parameter applied on every residual and every layer + tie_dropout: Tie dropout mask across sequence like nn.Dropout1d/nn.Dropout2d + prenorm: Pre-norm vs. post-norm + n_repeat: Each layer is repeated n times per stage before applying pooling + layer: Layer config, must be specified + residual: Residual config + norm: Normalization config (e.g. layer vs batch) + pool: Config for pooling layer per stage + track_norms: Log norms of each layer output + dropinp: Input dropout + drop_path: Stochastic depth for each residual path """ def __init__( @@ -133,40 +117,6 @@ def __init__( self.norm = nn.Identity() def forward(self, inputs, *args, state=None, **kwargs): - """ - Forward pass of the SequenceModel. - - This method processes the input through all layers of the model, applying - dropout, normalization, and tracking norms if specified. - - Args: - inputs (torch.Tensor): Input tensor of shape (batch, sequence, dim) or - (batch, dim, sequence) if self.transposed is True. - *args: Variable length argument list. - state (list, optional): List of previous states for each layer. If None, - default states will be used. Defaults to None. - **kwargs: Arbitrary keyword arguments. - - Returns: - tuple: A tuple containing: - - outputs (torch.Tensor): The processed output tensor of shape - (batch, sequence, d_output) or (batch, d_output, sequence) if - self.transposed is True. - - next_states (list): List of updated states for each layer. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4) - >>> inputs = torch.randn(32, 100, 256) - >>> outputs, states = model.forward(inputs) - >>> outputs.shape - torch.Size([32, 100, 256]) - >>> len(states) - 4 - - Note: - If self.track_norms is True, the method will update self.metrics with - the mean squared values of the outputs at each layer. - """ # Inputs assumed to be (batch, sequence, dim) if self.transposed: inputs = rearrange(inputs, "b ... d -> b d ...") @@ -199,57 +149,11 @@ def forward(self, inputs, *args, state=None, **kwargs): @property def d_state(self): - """ - Total dimension of the state across all layers. - - This property calculates and returns the sum of state dimensions for all layers - in the model that have a state. - - Returns: - int: The total dimension of the state across all layers. If a layer doesn't - have a state (i.e., its d_state is None), it's not included in the sum. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4) - >>> model.d_state - 1024 # Assuming each layer has a state dimension of 256 - - Note: - This property is useful for understanding the total state size of the model, - which can be important for memory considerations or when initializing or - processing the entire state of the model at once. - """ d_states = [layer.d_state for layer in self.layers] return sum([d for d in d_states if d is not None]) @property def state_to_tensor(self): - """ - Property that returns a function to convert model state to a tensor. - - This property provides a function that concatenates the tensor representations - of states from all layers into a single tensor. - - Returns: - function: A function that takes a state (list of layer states) as input - and returns a concatenated tensor of all non-None states. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4) - >>> state = model.default_state(batch_size=32) - >>> state_tensor = model.state_to_tensor(state) - >>> state_tensor.shape - torch.Size([32, 1024]) # Assuming each layer has a state dimension of 256 - - Note: - This property uses a closure to create a function that has access to - the model's layers. The returned function is "curried" in the sense that - it's partially applied to the model's layers and can be called later - with a state argument. - - The function skips any None states, only concatenating tensor states. - """ - # Slightly hacky way to implement this in a curried manner # (so that the function can be extracted from an instance) # Somewhat more sound may be to turn this into a @@ -265,75 +169,11 @@ def fn(state): return fn def default_state(self, *batch_shape, device=None): - """ - Generate the default initial state for the model. - - This method creates a list of default states for each layer in the model. - - Args: - *batch_shape (tuple): Variable length argument list for the batch dimensions. - device (torch.device, optional): The device on which to create the state tensors. - If None, uses the default device. Defaults to None. - - Returns: - list: A list of default states for each layer in the model. The structure and - content of each state depends on the specific implementation of each layer. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4) - >>> state = model.default_state(32) # for a batch size of 32 - >>> len(state) - 4 # one state per layer - >>> state = model.default_state(32, device=torch.device('cuda')) # on GPU - - Note: - The shape and content of the default state for each layer may vary depending - on the layer type and configuration. Some layers might return None if they - don't maintain a state. - - This method is particularly useful for initializing the model's state at the - beginning of a sequence or when no previous state is available. - """ return [ layer.default_state(*batch_shape, device=device) for layer in self.layers ] def step(self, x, state, **kwargs): - """ - Perform a single step forward pass of the model. - - This method applies the model to a single step of input, updating the state - for each layer in the process. - - Args: - x (torch.Tensor): Input tensor for a single step. The shape should be - compatible with the model's input requirements for a single time step. - state (list, optional): List of previous states for each layer. If None, - default states will be used. Defaults to None. - **kwargs: Additional keyword arguments to be passed to each layer's step method. - - Returns: - tuple: A tuple containing: - - x (torch.Tensor): The output tensor after processing through all layers. - - next_states (list): List of updated states for each layer. - - Examples: - >>> model = SequenceModel(d_model=256, n_layers=4) - >>> x = torch.randn(32, 256) # (batch_size, d_model) - >>> state = model.default_state(32) - >>> output, new_state = model.step(x, state) - >>> output.shape - torch.Size([32, 256]) - >>> len(new_state) - 4 - - Note: - This method is particularly useful for processing sequences one step at a time, - which can be more memory-efficient for very long sequences or in scenarios - where future inputs are not available, such as in real-time or streaming applications. - - The final normalization layer is applied to the output before returning. - """ # Apply layers prev_states = [None] * len(self.layers) if state is None else state next_states = [] diff --git a/espnet2/asr/state_spaces/pool.py b/espnet2/asr/state_spaces/pool.py index dc4e6f4e293..4d08194d770 100644 --- a/espnet2/asr/state_spaces/pool.py +++ b/espnet2/asr/state_spaces/pool.py @@ -18,35 +18,6 @@ def downsample(x, stride=1, expand=1, transposed=False): - """ - Downsample a sequence along the layer dimension and optionally expand along the feature dimension. - - This function performs downsampling on the input sequence by selecting elements at regular intervals - and optionally expands the feature dimension by repeating values. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - stride (int, optional): The stride for downsampling. Defaults to 1. - expand (int, optional): The expansion factor for the feature dimension. Defaults to 1. - transposed (bool, optional): Whether the input is in transposed format. Defaults to False. - - Returns: - torch.Tensor: Downsampled and optionally expanded tensor. - - Raises: - AssertionError: If the input tensor is not 3-dimensional when stride > 1. - - Examples: - >>> x = torch.randn(32, 100, 64) # (batch_size, seq_len, feature_dim) - >>> y = downsample(x, stride=2, expand=2) - >>> y.shape - torch.Size([32, 50, 128]) - - Note: - - If stride > 1, the function only supports 3-dimensional inputs. - - For higher-dimensional inputs with stride > 1, it is recommended to use average or spectral pooling instead. - """ if x is None: return None if stride > 1: @@ -69,32 +40,6 @@ def downsample(x, stride=1, expand=1, transposed=False): def upsample(x, stride=1, expand=1, transposed=False): - """ - Upsample a sequence along the layer dimension and optionally reduce along the feature dimension. - - This function performs upsampling on the input sequence by repeating elements and optionally - reduces the feature dimension by averaging values. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - stride (int, optional): The stride for upsampling. Defaults to 1. - expand (int, optional): The reduction factor for the feature dimension. Defaults to 1. - transposed (bool, optional): Whether the input is in transposed format. Defaults to False. - - Returns: - torch.Tensor: Upsampled and optionally reduced tensor. - - Examples: - >>> x = torch.randn(32, 50, 128) # (batch_size, seq_len, feature_dim) - >>> y = upsample(x, stride=2, expand=2) - >>> y.shape - torch.Size([32, 100, 64]) - - Note: - - If expand > 1, the function reduces the feature dimension by taking the mean of groups of 'expand' features. - - If stride > 1, the function repeats each element 'stride' times along the sequence dimension. - """ if x is None: return None if expand > 1: @@ -111,23 +56,6 @@ def upsample(x, stride=1, expand=1, transposed=False): class DownSample(SequenceModule): - """ - A module for downsampling sequences. - - This class implements downsampling on input sequences by selecting elements at regular intervals - and optionally expanding the feature dimension. It inherits from SequenceModule and can be used - in sequential models. - - Attributes: - d_input (int): The input feature dimension. - stride (int): The stride for downsampling. - expand (int): The expansion factor for the feature dimension. - transposed (bool): Whether the input is in transposed format. - - Note: - The `step` method is not implemented for stride > 1 or expand > 1. - """ - def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -136,90 +64,19 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): - """ - Forward pass for the DownSample module. - - This method applies downsampling to the input tensor using the specified stride and expand parameters. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - - Returns: - torch.Tensor: Downsampled tensor with shape depending on the stride and expand parameters. - - Note: - This method uses the `downsample` function internally, with `transposed=False` hardcoded. - The actual transposition behavior is controlled by the `self.transposed` attribute. - """ return downsample(x, self.stride, self.expand, False, self.transposed) def step(self, x, state, **kwargs): - """ - Perform a single step of the DownSample module. - - This method is intended for use in sequential or recurrent scenarios where the input is processed - one step at a time. - - Args: - x (torch.Tensor): Input tensor for the current step. - state: The current state of the module (unused in this implementation). - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor for the current step (same as input). - - state: The updated state (unchanged in this implementation). - - Raises: - NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. - - Note: - This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. - """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): - """ - int: The output feature dimension of the DownSample module. - - This property calculates and returns the output feature dimension based on the input dimension - and the expansion factor. - - Returns: - int: The output feature dimension, which is the product of the input dimension and the expand factor. - - Note: - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. - """ return self.d_input * self.expand class DownAvgPool(SequenceModule): - """ - A module for downsampling sequences using average pooling. - - This class implements downsampling on input sequences by applying average pooling - and optionally expanding the feature dimension. It inherits from SequenceModule and - can be used in sequential models. - - Attributes: - d_input (int): The input feature dimension. - stride (int): The stride for downsampling. - expand (int): The expansion factor for the feature dimension. - transposed (bool): Whether the input is in transposed format. - - Note: - - This module supports multi-dimensional inputs (e.g., 1D, 2D, or higher). - - The `step` method is not implemented for stride > 1 or expand > 1. - - Average pooling is applied along the sequence dimension(s) when stride > 1. - - Feature expansion is performed by repeating when expand > 1. - """ - def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -228,26 +85,6 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): - """ - Forward pass for the DownAvgPool module. - - This method applies average pooling for downsampling and optionally expands the feature dimension - of the input tensor. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, ..., feature_dim) if not transposed, - or (batch_size, feature_dim, ...) if transposed. - - Returns: - torch.Tensor: Downsampled and optionally expanded tensor. - - Note: - - If not transposed, the input is rearranged to (batch_size, feature_dim, ...) before processing. - - Average pooling is applied using F.avg_pool1d for 3D inputs, F.avg_pool2d for 4D inputs, - and a custom reduction for higher dimensions. - - If expand > 1, the feature dimension is expanded by repeating. - - The output is rearranged back to the original format if not transposed. - """ if not self.transposed: x = rearrange(x, "b ... d -> b d ...") @@ -275,75 +112,16 @@ def forward(self, x): return x def step(self, x, state, **kwargs): - """ - Perform a single step of the DownAvgPool module. - - This method is intended for use in sequential or recurrent scenarios where the input is processed - one step at a time. - - Args: - x (torch.Tensor): Input tensor for the current step. - state: The current state of the module (unused in this implementation). - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor for the current step (same as input). - - state: The updated state (unchanged in this implementation). - - Raises: - NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. - - Note: - This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. - """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): - """ - int: The output feature dimension of the DownAvgPool module. - - This property calculates and returns the output feature dimension based on the input dimension - and the expansion factor. - - Returns: - int: The output feature dimension, which is the product of the input dimension and the expand factor. - - Note: - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. The output dimension is expanded - if the `expand` attribute is greater than 1. - """ return self.d_input * self.expand class DownSpectralPool(SequenceModule): - """ - A module for downsampling sequences using spectral pooling. - - This class implements downsampling on input sequences by applying spectral pooling - in the frequency domain and optionally expanding the feature dimension. It inherits - from SequenceModule and can be used in sequential models. - - Spectral pooling is performed by truncating high-frequency components in the Fourier domain, - which can preserve more information compared to traditional spatial pooling methods. - - Attributes: - d_input (int): The input feature dimension. - stride (int): The stride for downsampling. - expand (int): The expansion factor for the feature dimension. - transposed (bool): Whether the input is in transposed format. - - Note: - - This module supports multi-dimensional inputs. - - The `step` method is not implemented for stride > 1 or expand > 1. - - Spectral pooling is applied along all spatial dimensions when stride > 1. - - Feature expansion is performed by repeating when expand > 1. - """ - def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -352,30 +130,9 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): - """ - Forward pass for the DownSpectralPool module. - - This method applies spectral pooling for downsampling and optionally expands the feature dimension - of the input tensor. The spectral pooling is performed in the frequency domain using FFT. - - Args: - x (torch.Tensor): Input tensor of shape (B, L..., D) where B is batch size, L... are spatial dimensions, - and D is the feature dimension. - - Returns: - torch.Tensor: Downsampled and optionally expanded tensor. - - Raises: - AssertionError: If any spatial dimension length is not divisible by the stride. + """Forward pass. - Note: - - If not transposed, the input is rearranged to (B, D, L...) before processing. - - The method performs the following steps: - 1. Applies FFT to the input tensor. - 2. Truncates high-frequency components based on the stride. - 3. Applies inverse FFT to return to the spatial domain. - 4. Expands the feature dimension if expand > 1. - - The output is rearranged back to the original format if not transposed. + x: (B, L..., D) """ if not self.transposed: x = rearrange(x, "b ... d -> b d ...") @@ -399,73 +156,16 @@ def forward(self, x): return x def step(self, x, state, **kwargs): - """ - Perform a single step of the DownSpectralPool module. - - This method is intended for use in sequential or recurrent scenarios where the input is processed - one step at a time. - - Args: - x (torch.Tensor): Input tensor for the current step. - state: The current state of the module (unused in this implementation). - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor for the current step (same as input). - - state: The updated state (unchanged in this implementation). - - Raises: - NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. - - Note: - This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. - Spectral pooling is not performed in the step method, as it requires processing the entire sequence. - """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): - """ - int: The output feature dimension of the DownSpectralPool module. - - This property calculates and returns the output feature dimension based on the input dimension - and the expansion factor. - - Returns: - int: The output feature dimension, which is the product of the input dimension and the expand factor. - - Note: - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. The output dimension is expanded - if the `expand` attribute is greater than 1, while the spatial dimensions are reduced based - on the `stride` attribute. - """ return self.d_input * self.expand class UpSample(nn.Module): - """ - A module for upsampling sequences. - - This class implements upsampling on input sequences by repeating elements along the sequence dimension - and optionally reducing the feature dimension. It inherits from nn.Module and can be used in - neural network architectures. - - Attributes: - d_input (int): The input feature dimension. - stride (int): The stride for upsampling (number of times each element is repeated). - expand (int): The reduction factor for the feature dimension. - transposed (bool): Whether the input is in transposed format. - - Note: - - The actual upsampling is performed in the forward method using the `upsample` function. - - The `step` method is not implemented for stride > 1 or expand > 1. - - The output dimension is calculated as d_input // expand, effectively reducing the feature dimension. - """ - def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() self.d_input = d_input @@ -474,69 +174,13 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): self.transposed = transposed def forward(self, x): - """ - Forward pass for the UpSample module. - - This method applies upsampling to the input tensor using the specified stride and expand parameters. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - - Returns: - torch.Tensor: Upsampled tensor with shape depending on the stride and expand parameters. - - Note: - This method uses the `upsample` function internally. The upsampling process involves: - 1. Repeating elements along the sequence dimension based on the `stride` parameter. - 2. Reducing the feature dimension based on the `expand` parameter (if > 1). - The exact behavior depends on whether the input is in transposed format or not. - """ return upsample(x, self.stride, self.expand, self.transposed) @property def d_output(self): - """ - int: The output feature dimension of the UpSample module. - - This property calculates and returns the output feature dimension based on the input dimension - and the expand factor. - - Returns: - int: The output feature dimension, which is the input dimension divided by the expand factor. - - Note: - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. The output dimension is reduced - if the `expand` attribute is greater than 1, reflecting the feature dimension reduction - that occurs during upsampling. - """ return self.d_input // self.expand def step(self, x, state, **kwargs): - """ - Perform a single step of the UpSample module. - - This method is intended for use in sequential or recurrent scenarios where the input is processed - one step at a time. - - Args: - x (torch.Tensor): Input tensor for the current step. - state: The current state of the module (unused in this implementation). - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor for the current step (same as input). - - state: The updated state (unchanged in this implementation). - - Raises: - NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. - - Note: - This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. - Upsampling is not performed in the step method, as it requires processing multiple time steps at once. - """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @@ -547,26 +191,6 @@ def step(self, x, state, **kwargs): class DownLinearPool(SequenceModule): - """ - A module for downsampling sequences using linear pooling. - - This class implements downsampling on input sequences by applying a linear transformation - to groups of elements and optionally expanding the feature dimension. It inherits from - SequenceModule and can be used in sequential models. - - Attributes: - d_input (int): The input feature dimension. - stride (int): The stride for downsampling (number of elements to group). - expand (int): The expansion factor for the feature dimension. - transposed (bool): Whether the input is in transposed format. - linear (LinearActivation): The linear transformation applied to grouped elements. - - Note: - - The linear transformation is applied to groups of `stride` elements along the sequence dimension. - - The output feature dimension is expanded by a factor of `expand`. - - The `step` method is not implemented for stride > 1 or expand > 1. - """ - def __init__(self, d_input, stride=1, expand=1, transposed=True): super().__init__() @@ -582,25 +206,6 @@ def __init__(self, d_input, stride=1, expand=1, transposed=True): ) def forward(self, x): - """ - Forward pass for the DownLinearPool module. - - This method applies linear pooling for downsampling and optionally expands the feature dimension - of the input tensor. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - - Returns: - torch.Tensor: Downsampled and optionally expanded tensor. - - Note: - - If transposed, the input is rearranged to group `stride` elements along the last dimension. - - If not transposed, the input is rearranged to group `stride` elements along the second-to-last dimension. - - The linear transformation is applied to the grouped elements, effectively downsampling the sequence. - - The output tensor's shape depends on the stride, expand, and transposed parameters. - """ if self.transposed: x = rearrange(x, "... h (l s) -> ... (h s) l", s=self.stride) else: @@ -609,50 +214,12 @@ def forward(self, x): return x def step(self, x, state, **kwargs): - """ - Perform a single step of the DownLinearPool module. - - This method is intended for use in sequential or recurrent scenarios where the input is processed - one step at a time. - - Args: - x (torch.Tensor): Input tensor for the current step. - state: The current state of the module (unused in this implementation). - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: The output tensor for the current step (same as input). - - state: The updated state (unchanged in this implementation). - - Raises: - NotImplementedError: If stride > 1 or expand > 1, as these cases are not supported for step-wise processing. - - Note: - This method currently only supports stride=1 and expand=1. For other values, it raises a NotImplementedError. - Linear pooling is not performed in the step method, as it requires processing multiple time steps at once. - """ if self.stride > 1 or self.expand > 1: raise NotImplementedError return x, state @property def d_output(self): - """ - int: The output feature dimension of the DownLinearPool module. - - This property calculates and returns the output feature dimension based on the input dimension - and the expansion factor. - - Returns: - int: The output feature dimension, which is the product of the input dimension and the expand factor. - - Note: - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. The output dimension is expanded - by the `expand` factor, reflecting the feature dimension expansion that occurs during - the linear pooling operation. - """ return self.d_input * self.expand @@ -660,23 +227,6 @@ def d_output(self): class DownPool2d(SequenceModule): - """ - A module for downsampling 2D sequences using a combination of linear transformation and average pooling. - - This class implements downsampling on 2D input sequences by first applying a linear transformation - to change the feature dimension, followed by average pooling to reduce spatial dimensions. It inherits - from SequenceModule and can be used in sequential models dealing with 2D data. - - Attributes: - linear (LinearActivation): Linear transformation applied to the input before pooling. - pool (nn.AvgPool2d): Average pooling layer for spatial downsampling. - - Note: - - The linear transformation allows for changing the feature dimension before pooling. - - The average pooling operation reduces the spatial dimensions based on the given stride. - - This module is particularly useful for processing 2D data such as images or feature maps. - """ - def __init__(self, d_input, d_output, stride=1, transposed=True, weight_norm=True): super().__init__() @@ -690,23 +240,6 @@ def __init__(self, d_input, d_output, stride=1, transposed=True, weight_norm=Tru self.pool = (nn.AvgPool2d(kernel_size=stride, stride=stride),) def forward(self, x): - """ - Forward pass for the DownPool2d module. - - This method applies the linear transformation followed by average pooling to downsample the input tensor. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, channels, height, width) if transposed, - or (batch_size, height, width, channels) if not transposed. - - Returns: - torch.Tensor: Downsampled tensor after linear transformation and pooling. - - Note: - - If transposed, the pooling operation is applied directly after the linear transformation. - - The implementation for non-transposed input is not provided in the given code snippet. - - The output tensor's spatial dimensions will be reduced based on the pooling parameters. - """ if self.transposed: x = self.pool(x) @@ -714,26 +247,6 @@ def forward(self, x): # DownLinearPool is used by the registry (for isotropic backbone) # DownPool is essentially the same as DownLinearPool. These should be consolidated class DownPool(SequenceModule): - """ - A flexible module for downsampling sequences using linear transformation. - - This class implements downsampling on input sequences by applying a linear transformation - to groups of elements. It allows for both downsampling and feature dimension adjustment. - It inherits from SequenceModule and can be used in sequential models. - - Attributes: - d_output (int): The output feature dimension. - stride (int): The stride for downsampling (number of elements to group). - transposed (bool): Whether the input is in transposed format. - linear (LinearActivation): The linear transformation applied to grouped elements. - - Note: - - The linear transformation is applied to groups of `stride` elements along the sequence dimension. - - This module can both downsample the sequence and adjust the feature dimension. - - Supports optional weight normalization and activation function in the linear transformation. - - The `step` method provides functionality for recurrent processing. - """ - def __init__( self, d_input, @@ -765,26 +278,6 @@ def __init__( ) def forward(self, x): - """ - Forward pass for the DownPool module. - - This method applies downsampling and linear transformation to the input tensor. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Downsampled and transformed tensor. - - None: Placeholder for consistency with other modules that might return additional information. - - Note: - - If transposed, the input is rearranged to group `stride` elements along the second dimension. - - If not transposed, the input is rearranged to group `stride` elements along the second-to-last dimension. - - The linear transformation is applied to the grouped elements, effectively downsampling the sequence. - - The output tensor's shape depends on the stride, d_output, and transposed parameters. - """ if self.transposed: x = rearrange(x, "... h (l s) -> ... (h s) l", s=self.stride) else: @@ -793,27 +286,9 @@ def forward(self, x): return x, None def step(self, x, state, **kwargs): - """ - Perform a single step of the DownPool module for recurrent processing. - - This method allows the DownPool module to be used in a recurrent manner, processing - one time step at a time and maintaining an internal state. - - Args: - x (torch.Tensor or None): Input tensor for the current step, or None if no input is available. - state (list): Current state of the module, containing previously seen inputs. - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor or None: Output tensor if a full group is processed, otherwise None. - - list: Updated state of the module. - - Note: - - The method accumulates inputs in the state until it has gathered 'stride' elements. - - When 'stride' elements are gathered, it applies the linear transformation and resets the state. - - If x is None, it returns None and the current state. - - This method enables the module to work with variable-length sequences in a recurrent setting. + """Step one time step as a recurrent model. + + x: (..., H) """ if x is None: return None, state @@ -830,49 +305,10 @@ def step(self, x, state, **kwargs): return None, state def default_state(self, *batch_shape, device=None): - """ - Initialize the default state for the DownPool module. - - This method creates the initial state for the module when used in a recurrent setting. - - Args: - *batch_shape: Variable length argument to specify the batch dimensions. - device (torch.device, optional): The device on which to create the state. Defaults to None. - - Returns: - list: An empty list representing the initial state of the module. - - Note: - - The default state is an empty list, which will be filled with input tensors during the step method. - - This method is typically called once before starting the recurrent processing of a sequence. - - The batch_shape and device arguments are included for compatibility with other modules - but are not used in this implementation. - """ return [] class UpPool(SequenceModule): - """ - A module for upsampling sequences using linear transformation. - - This class implements upsampling on input sequences by applying a linear transformation - followed by reshaping to increase the sequence length. It inherits from SequenceModule - and can be used in sequential models. - - Attributes: - d_input (int): The input feature dimension. - _d_output (int): The output feature dimension. - stride (int): The upsampling factor (increase in sequence length). - transposed (bool): Whether the input is in transposed format. - linear (LinearActivation): The linear transformation applied before upsampling. - - Note: - - The linear transformation expands the feature dimension by a factor of `stride`. - - The output is then reshaped to increase the sequence length by `stride`. - - Supports optional weight normalization and activation function in the linear transformation. - - Includes a step method for recurrent processing and a default_state method for initialization. - """ - def __init__( self, d_input, @@ -901,28 +337,6 @@ def __init__( ) def forward(self, x, skip=None): - """ - Forward pass for the UpPool module. - - This method applies upsampling to the input tensor using linear transformation and reshaping. - - Args: - x (torch.Tensor): Input tensor of shape (batch_size, seq_len, feature_dim) if not transposed, - or (batch_size, feature_dim, seq_len) if transposed. - skip (torch.Tensor, optional): Skip connection input to be added after upsampling. Defaults to None. - - Returns: - tuple: A tuple containing: - - torch.Tensor: Upsampled tensor, potentially with skip connection added. - - None: Placeholder for consistency with other modules that might return additional information. - - Note: - - The input is first passed through a linear transformation to expand the feature dimension. - - The transformed tensor is then reshaped to increase the sequence length by the stride factor. - - A shift operation is applied to ensure causality in the upsampled sequence. - - If a skip connection is provided, it's added to the upsampled tensor. - - The output tensor's shape depends on the stride, d_output, and transposed parameters. - """ x = self.linear(x) if self.transposed: x = F.pad(x[..., :-1], (1, 0)) # Shift to ensure causality @@ -935,27 +349,9 @@ def forward(self, x, skip=None): return x, None def step(self, x, state, **kwargs): - """ - Perform a single step of the UpPool module for recurrent processing. - - This method allows the UpPool module to be used in a recurrent manner, processing - one time step at a time and maintaining an internal state. - - Args: - x (torch.Tensor or None): Input tensor for the current step, or None if no input is available. - state (list): Current state of the module, containing future outputs. - **kwargs: Additional keyword arguments (unused in this implementation). - - Returns: - tuple: A tuple containing: - - torch.Tensor: Output tensor for the current step. - - list: Updated state of the module. - - Note: - - The method returns the next output from the state and updates the state. - - If the state is empty and x is provided, it applies the linear transformation and generates new outputs. - - This method enables the module to work with variable-length sequences in a recurrent setting. - - It maintains causality by only using current and past inputs to generate outputs. + """Step one time step as a recurrent model. + + x: (..., H) """ assert len(state) > 0 y, state = state[0], state[1:] @@ -973,25 +369,6 @@ def step(self, x, state, **kwargs): return y, state def default_state(self, *batch_shape, device=None): - """ - Initialize the default state for the UpPool module. - - This method creates the initial state for the module when used in a recurrent setting. - - Args: - *batch_shape: Variable length argument to specify the batch dimensions. - device (torch.device, optional): The device on which to create the state. Defaults to None. - - Returns: - list: A list of torch.Tensor objects representing the initial state of the module. - - Note: - - The default state is a list of zero tensors, with shape (batch_shape, d_output, stride). - - Each tensor in the list represents a future output in the upsampled sequence. - - The state is initialized with 'stride' number of zero tensors. - - This method is typically called once before starting the recurrent processing of a sequence. - - The state allows the module to maintain the expanded sequence length during step-wise processing. - """ state = torch.zeros( batch_shape + (self.d_output, self.stride), device=device ) # (batch, h, s) @@ -1000,20 +377,6 @@ def default_state(self, *batch_shape, device=None): @property def d_output(self): - """ - int: The output feature dimension of the UpPool module. - - This property returns the output feature dimension of the module. - - Returns: - int: The output feature dimension. - - Note: - - This property returns the value of the `_d_output` attribute, which is set during initialization. - - The output dimension is typically different from the input dimension due to the upsampling process. - - This property is useful for determining the output shape of the module, especially when - chaining multiple modules together in a sequential model. - """ return self._d_output diff --git a/espnet2/asr/state_spaces/residual.py b/espnet2/asr/state_spaces/residual.py index 472bf901c9d..bcadadb02f1 100644 --- a/espnet2/asr/state_spaces/residual.py +++ b/espnet2/asr/state_spaces/residual.py @@ -7,37 +7,9 @@ class Residual(nn.Module): - """ - Residual connection with constant affine weights. - - This class implements a residual connection that can simulate standard residual, - no residual, and "constant gates" behaviors. - - Attributes: - i_layer (int): The index of the current layer. - d_input (int): The dimension of the input. - d_model (int): The dimension of the model. - alpha (float): The weight for the input in the residual connection. - beta (float): The weight for the transformed input in the residual connection. - - Args: - i_layer (int): The index of the current layer. - d_input (int): The dimension of the input. - d_model (int): The dimension of the model. - alpha (float, optional): The weight for the input. Defaults to 1.0. - beta (float, optional): The weight for the transformed input. Defaults to 1.0. + """Residual connection with constant affine weights. - Note: - The input dimension (d_input) must be equal to the model dimension (d_model) - unless alpha is set to 0.0. - - Examples: - >>> residual = Residual(1, 64, 64) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = residual(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) + Can simulate standard residual, no residual, and "constant gates". """ def __init__(self, i_layer, d_input, d_model, alpha=1.0, beta=1.0): @@ -52,89 +24,18 @@ def __init__(self, i_layer, d_input, d_model, alpha=1.0, beta=1.0): @property def d_output(self): - """ - Returns the output dimension of the residual connection. - - Returns: - int: The dimension of the model (d_model), which is the output dimension - of the residual connection. - - Note: - This property always returns the d_model attribute, as the residual - connection maintains the same dimension as the input model. - - Examples: - >>> residual = Residual(1, 64, 64) - >>> residual.d_output - 64 - """ return self.d_model def forward(self, x, y, transposed): - """ - Performs the forward pass of the residual connection. - - This method applies the residual connection by combining the input x - and the transformed input y using the weights alpha and beta. - - Args: - x (torch.Tensor): The input tensor. - y (torch.Tensor): The transformed input tensor. - transposed (bool): A flag indicating whether the input is transposed. - This argument is not used in the current implementation but is - included for consistency with other similar methods. - - Returns: - torch.Tensor: The output of the residual connection. - - Note: - The formula used is: output = alpha * x + beta * y - If beta is 1.0, y is used as is. If alpha is 0, only y is returned. - - Examples: - >>> residual = Residual(1, 64, 64, alpha=0.5, beta=1.5) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = residual.forward(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ y = self.beta * y if self.beta != 1.0 else y return self.alpha * x + y if self.alpha else y class Affine(Residual): - """ - Residual connection with learnable scalar multipliers on the main branch. - - This class extends the Residual class by adding learnable scalar multipliers - to the main branch of the residual connection. It can be initialized with - either a single scalar multiplier or one per dimension. + """Residual connection with learnable scalar multipliers on the main branch. - Attributes: - scalar (bool): If True, use a single scalar multiplier; otherwise, use one per dimension. - gamma (float): Power factor for layer-dependent scaling initialization. - affine (nn.Parameter): Learnable scalar multiplier(s) for the main branch. - - Args: - *args: Variable length argument list passed to the parent Residual class. - scalar (bool, optional): Whether to use a single scalar multiplier. Defaults to True. - gamma (float, optional): Power factor for layer-dependent scaling. Defaults to 0.0. - **kwargs: Arbitrary keyword arguments passed to the parent Residual class. - - Note: - The affine parameter is initialized as: - c * torch.ones(d), where - c = beta * i_layer ** (-gamma) - d = 1 if scalar is True, else d_input - - Examples: - >>> affine_residual = Affine(1, 64, 64, scalar=False, gamma=0.1) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = affine_residual(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) + scalar: Single scalar multiplier, or one per dimension + scale, power: Initialize to scale * layer_num**(-power) """ def __init__(self, *args, scalar=True, gamma=0.0, **kwargs): @@ -148,33 +49,6 @@ def __init__(self, *args, scalar=True, gamma=0.0, **kwargs): self.affine = nn.Parameter(c * torch.ones(d)) def forward(self, x, y, transposed): - """ - Performs the forward pass of the Affine residual connection. - - This method applies the residual connection by combining the input x - and the transformed input y using the learnable affine parameter. - - Args: - x (torch.Tensor): The input tensor. - y (torch.Tensor): The transformed input tensor. - transposed (bool): A flag indicating whether the input is transposed. - If True, the affine parameter is unsqueezed along the last dimension. - - Returns: - torch.Tensor: The output of the Affine residual connection. - - Note: - The formula used is: output = alpha * x + c * y - where c is the learnable affine parameter. - - Examples: - >>> affine_residual = Affine(1, 64, 64, scalar=False, gamma=0.1) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = affine_residual.forward(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ c = self.affine if transposed: c = c.unsqueeze(-1) @@ -182,72 +56,12 @@ def forward(self, x, y, transposed): class Feedforward(Residual): - """ - A feedforward residual connection. - - This class implements a simple feedforward residual connection where the - input x is ignored, and only the transformed input y is passed through. - It's a special case of the Residual class with alpha set to 0.0 and beta set to 1.0. - - Args: - *args: Variable length argument list passed to the parent Residual class. - - Note: - This class effectively removes the residual connection, as it only - passes through the transformed input y without adding the original input x. - - Examples: - >>> feedforward = Feedforward(1, 64, 64) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = feedforward(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - >>> torch.allclose(output, y) - True - """ - def __init__(self, *args): # print("Feedforward extra kwargs", kwargs) super().__init__(*args, alpha=0.0, beta=1.0) class Highway(Residual): - """ - Implements a Highway residual connection. - - This class extends the Residual class to create a Highway network connection, - which allows the model to adaptively choose between passing information - through a nonlinear transformation or passing it unchanged. - - Attributes: - scaling_correction (float): A scaling factor to adjust the output. - elemwise (bool): If True, uses element-wise multiplication for Wy; - otherwise, uses a linear transformation. - Wx (nn.Linear): Linear transformation for the input x. - Wy (nn.Parameter or nn.Linear): Transformation for the input y, - either element-wise or linear. - - Args: - *args: Variable length argument list passed to the parent Residual class. - scaling_correction (bool, optional): If True, applies a scaling correction - factor of 1.732. Defaults to False. - elemwise (bool, optional): If True, uses element-wise multiplication - for Wy. Defaults to False. - - Note: - The Highway connection computes a gating mechanism to control - information flow from the input and transformed input. - - Examples: - >>> highway = Highway(1, 64, 64, scaling_correction=True, elemwise=False) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = highway(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ - def __init__(self, *args, scaling_correction=False, elemwise=False): super().__init__(*args) self.scaling_correction = 1.732 if scaling_correction else 1.0 @@ -259,38 +73,6 @@ def __init__(self, *args, scaling_correction=False, elemwise=False): self.Wy = nn.Linear(self.d_input, self.d_input) def forward(self, x, y, transposed=False): - """ - Performs the forward pass of the Highway residual connection. - - This method implements the Highway network mechanism, which adaptively - combines the original input and its transformation using a gating function. - - Args: - x (torch.Tensor): The original input tensor. - y (torch.Tensor): The transformed input tensor. - transposed (bool): A flag indicating whether the input is transposed. - This argument is not used in the current implementation but is - included for consistency with other similar methods. - - Returns: - torch.Tensor: The output of the Highway residual connection. - - Note: - The Highway mechanism is computed as follows: - 1. Compute the gate: r = sigmoid(Wx(x) + Wy(y)) - 2. Combine inputs: z = scaling_correction * (1 - r) * x + r * y - - If elemwise is True, Wy is applied as element-wise multiplication. - Otherwise, it's applied as a linear transformation. - - Examples: - >>> highway = Highway(1, 64, 64, scaling_correction=True, elemwise=False) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = highway.forward(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ if self.elemwise: y = self.Wy * y else: @@ -301,34 +83,7 @@ def forward(self, x, y, transposed=False): class DecayResidual(Residual): - """ - Implements a residual connection with depth-dependent decay. - - This class extends the Residual class to create a connection where the - contribution of the residual path decays based on the depth of the layer. - - Attributes: - power (float): The power factor used in the decay calculation. - l2 (bool): If True, uses L2 normalization for alpha calculation; - otherwise, uses linear decay. - - Args: - *args: Variable length argument list passed to the parent Residual class. - power (float, optional): The power factor for decay calculation. Defaults to 0.5. - l2 (bool, optional): Whether to use L2 normalization. Defaults to True. - - Note: - The decay is calculated based on the layer index (i_layer) and the power factor. - The contribution of the original input decreases as the network depth increases. - - Examples: - >>> decay_residual = DecayResidual(1, 64, 64, power=0.5, l2=True) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = decay_residual(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ + """Residual connection that can decay the linear combination depending on depth.""" def __init__(self, *args, power=0.5, l2=True): # print("DecayResidual extra kwargs", kwargs) @@ -337,42 +92,6 @@ def __init__(self, *args, power=0.5, l2=True): self.l2 = l2 def forward(self, x, y, transposed): - """ - Performs the forward pass of the DecayResidual connection. - - This method implements the depth-dependent decay mechanism for combining - the original input and its transformation. - - Args: - x (torch.Tensor): The original input tensor. - y (torch.Tensor): The transformed input tensor. - transposed (bool): A flag indicating whether the input is transposed. - This argument is not used in the current implementation but is - included for consistency with other similar methods. - - Returns: - torch.Tensor: The output of the DecayResidual connection. - - Note: - The decay mechanism is computed as follows: - 1. Calculate beta: beta = i_layer ** (-power) - 2. Calculate alpha: - - If l2 is True: alpha = sqrt(1 - beta^2) - - If l2 is False: alpha = 1 - beta - 3. Combine inputs: output = alpha * x + beta * y - - As the layer index increases, beta decreases, reducing the contribution - of the transformed input y and increasing the contribution of the - original input x. - - Examples: - >>> decay_residual = DecayResidual(1, 64, 64, power=0.5, l2=True) - >>> x = torch.randn(10, 64) - >>> y = torch.randn(10, 64) - >>> output = decay_residual.forward(x, y, transposed=False) - >>> output.shape - torch.Size([10, 64]) - """ beta = self.i_layer ** (-self.power) if self.l2: alpha = (1.0 - beta**2) ** 0.5 diff --git a/espnet2/asr/state_spaces/s4.py b/espnet2/asr/state_spaces/s4.py index 123de939198..b29d7b9cc48 100644 --- a/espnet2/asr/state_spaces/s4.py +++ b/espnet2/asr/state_spaces/s4.py @@ -24,28 +24,10 @@ def rank_zero_only(fn: Callable) -> Callable: - """ - Decorator to make a function or method run only on the main process in distributed training. - - This decorator is useful for operations that should only be performed once - across all processes, such as logging or saving checkpoints. - - Args: - fn (Callable): The function to be decorated. - - Returns: - Callable: A wrapped version of the input function that will only execute - when the global rank is 0. + """Decorator function from PyTorch Lightning. - Example: - @rank_zero_only - def log_something(): - print("This will only be printed on the main process") - - Note: - This function assumes that the global rank can be determined from - environment variables. It checks for 'RANK', 'LOCAL_RANK', - 'SLURM_PROCID', and 'JSM_NAMESPACE_RANK' in that order. + Function that can be used as a decorator + to enable a function/method being called only on global rank 0. """ @wraps(fn) @@ -74,31 +56,7 @@ def _get_rank() -> int: def get_logger(name=__name__, level=logging.INFO) -> logging.Logger: - """ - Initialize a multi-GPU-friendly Python logger. - - This function creates a logger that is aware of distributed training environments. - It decorates all logging methods with the rank_zero_only decorator to ensure - that log messages are only printed once in multi-GPU setups. - - Args: - name (str, optional): The name of the logger. Defaults to the name of the - current module. - level (int, optional): The logging level. Defaults to logging.INFO. - - Returns: - logging.Logger: A configured logger instance with rank-zero-only decorated - logging methods. - - Example: - logger = get_logger("my_module") - logger.info("This message will only be logged once in multi-GPU training") - - Note: - This function modifies the behavior of the standard logging methods - (debug, info, warning, error, exception, fatal, critical) to only execute - on the main process (rank zero) in distributed environments. - """ + """Initialize multi-GPU-friendly python logger.""" logger = logging.getLogger(name) logger.setLevel(level) @@ -302,33 +260,10 @@ def _resolve_conj(x): def power(L, A, v=None): - """ - Compute A^L and the scan sum_i A^i v_i. - - This function calculates the matrix power A^L and optionally computes - the sum of A^i * v_i for i from 0 to L-1, where v_i is the i-th column of v. - - Args: - L (int): The power to raise A to. - A (torch.Tensor): Square matrix of shape (..., N, N). - v (torch.Tensor, optional): Tensor of shape (..., N, L). If provided, - the function also computes the scan sum. Defaults to None. - - Returns: - torch.Tensor or Tuple[torch.Tensor, torch.Tensor]: - If v is None, returns A^L of shape (..., N, N). - If v is provided, returns a tuple (A^L, sum_i A^i v_i) where - sum_i A^i v_i has shape (..., N). - - Note: - This function uses a divide-and-conquer strategy to compute the - matrix power efficiently. It's particularly useful for large L values. - - Example: - A = torch.randn(3, 3) - v = torch.randn(3, 5) - result = power(5, A, v) - # result is a tuple (A^5, sum of A^i * v_i for i from 0 to 4) + """Compute A^L and the scan sum_i A^i v_i. + + A: (..., N, N) + v: (..., N, L) """ E = torch.eye(A.shape[-1]).to(A) # , dtype=A.dtype, device=A.device) @@ -374,37 +309,7 @@ def power(L, A, v=None): def transition(measure, N): - """ - Compute the A and B transition matrices for different measures. - - This function generates the A and B matrices used in state space models - for various types of measures or basis functions. - - Args: - measure (str): The type of measure to use. Options include: - 'legt' (Legendre (translated)), - 'legs' (Legendre (scaled)), - 'legsd' (Scaled Legendre with diagonal correction), - 'fourier_diag' or 'foud' (Fourier diagonal), - 'fourier' or 'fout' (Fourier). - N (int): The size of the state space (number of basis functions). - - Returns: - Tuple[np.ndarray, np.ndarray]: A tuple containing: - - A (np.ndarray): The transition matrix of shape (N, N). - - B (np.ndarray): The input matrix of shape (N, 1). - - Raises: - NotImplementedError: If an unsupported measure is specified. - - Note: - The matrices are returned as NumPy arrays and may need to be - converted to PyTorch tensors for use in neural network modules. - - Example: - A, B = transition('legt', 64) - # A is a (64, 64) matrix, B is a (64, 1) matrix - """ + """A, B transition matrices for different measures.""" # Legendre (translated) if measure == "legt": Q = np.arange(N, dtype=np.float64) @@ -470,37 +375,7 @@ def transition(measure, N): def rank_correction(measure, N, rank=1, dtype=torch.float): - """ - Return a low-rank matrix L such that A + L is normal. - - This function computes a low-rank correction matrix for different types of - measures used in state space models. The correction ensures that A + L - has the desired normality properties. - - Args: - measure (str): The type of measure. Options include: - 'legs' (Legendre scaled), - 'legt' (Legendre translated), - 'fourier' or 'fout' (Fourier), - 'fourier_diag', 'foud', or 'legsd' (Diagonal corrections). - N (int): The size of the state space. - rank (int, optional): The rank of the correction matrix. Defaults to 1. - dtype (torch.dtype, optional): The data type of the output. Defaults to torch.float. - - Returns: - torch.Tensor: A low-rank correction matrix of shape (rank, N). - - Raises: - NotImplementedError: If an unsupported measure is specified. - - Note: - The returned matrix P is such that PP^* is the desired low-rank - correction. The actual correction is applied as A + PP^*. - - Example: - P = rank_correction('legs', 64, rank=2) - # P is a (2, 64) matrix - """ + """Return low-rank matrix L such that A + L is normal.""" if measure == "legs": assert rank >= 1 P = torch.sqrt(0.5 + torch.arange(N, dtype=dtype)).unsqueeze(0) # (1 N) @@ -532,39 +407,11 @@ def rank_correction(measure, N, rank=1, dtype=torch.float): def nplr(measure, N, rank=1, dtype=torch.float, diagonalize_precision=True): - """ - Decompose the HiPPO matrix into Normal Plus Low-Rank (NPLR) form. - - This function computes the NPLR decomposition of the HiPPO matrix for a given measure. - The decomposition is of the form A = V[w - p q^*]V^*, B = V B, where w is diagonal. - - Args: - measure (str): The type of HiPPO measure (e.g., 'legs', 'legt', 'fourier'). - N (int): The size of the state space. - rank (int, optional): The rank of the low-rank correction. Defaults to 1. - dtype (torch.dtype, optional): The data type for the computations. Defaults to torch.float. - diagonalize_precision (bool, optional): Whether to use higher precision for diagonalization. Defaults to True. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: - - w (torch.Tensor): The diagonal part of the decomposition. - - P (torch.Tensor): The low-rank factor p. - - B (torch.Tensor): The transformed input matrix. - - V (torch.Tensor): The transformation matrix. - - Raises: - NotImplementedError: If the specified measure is not implemented. - - Note: - This function is crucial for the efficient implementation of the S4 model. - The returned matrices satisfy the relation A = V[w - p q^*]V^*, B = V B. - - Example: - w, P, B, V = nplr('legs', 64, rank=2) - # w is a complex vector of length 32 - # P is a (2, 64) matrix - # B is a vector of length 64 - # V is a (64, 64) unitary matrix + """Decompose as Normal Plus Low-Rank (NPLR). + + Return w, p, q, V, B such that + (w - p q^*, B) is unitarily equivalent to the original HiPPO A, B by the matrix V + i.e. A = V[w - p q^*]V^*, B = V B """ assert dtype == torch.float or torch.double cdtype = torch.cfloat if dtype == torch.float else torch.cdouble @@ -645,44 +492,6 @@ def dplr( diagonal=True, random_B=False, ): - """ - Generate parameters for the Diagonal Plus Low-Rank (DPLR) parameterization of SSMs. - - This function creates the parameters needed for the DPLR version of the S4 model, - which uses a diagonal matrix plus a low-rank correction. - - Args: - scaling (str): The type of scaling to use for the imaginary part ('random', 'real', 'linear', 'inverse', 'inverse2', 'quadratic', 'legs', 'hippo'). - N (int): Size of the state space (half the size due to complex conjugacy). - rank (int, optional): Rank of the low-rank component. Defaults to 1. - H (int, optional): Number of independent SSM copies. Defaults to 1. - dtype (torch.dtype, optional): Data type of the generated tensors. Defaults to torch.float. - real_scale (float, optional): Scaling factor for the real part. Defaults to 1.0. - imag_scale (float, optional): Scaling factor for the imaginary part. Defaults to 1.0. - random_real (bool, optional): Whether to use random initialization for the real part. Defaults to False. - random_imag (bool, optional): Whether to use random initialization for the imaginary part. Defaults to False. - normalize (bool, optional): Whether to normalize the B vector. Defaults to False. - diagonal (bool, optional): Whether to use a diagonal matrix for P. Defaults to True. - random_B (bool, optional): Whether to use random initialization for B. Defaults to False. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: - - w (torch.Tensor): Complex diagonal coefficients of shape (H, N). - - P (torch.Tensor): Low-rank term of shape (rank, H, N). - - B (torch.Tensor): Input projection vector of shape (H, N). - - V (torch.Tensor): Transformation matrix of shape (H, N, N). - - Note: - This function is used to initialize the S4D model, which is a simplification - of the full S4 model using diagonal state matrices. - - Example: - w, P, B, V = dplr('linear', 64, rank=2, H=4) - # w is a complex tensor of shape (4, 32) - # P is a complex tensor of shape (2, 4, 32) - # B is a complex tensor of shape (4, 32) - # V is a complex tensor of shape (4, 64, 64) - """ assert dtype == torch.float or torch.double dtype = torch.cfloat if dtype == torch.float else torch.cdouble @@ -747,41 +556,11 @@ def dplr( def ssm(measure, N, R, H, **ssm_args): - """ - Create a single State Space Model (SSM) initialization. - - This function serves as a dispatcher to create SSM parameters based on the - specified measure. It supports both NPLR (Normal Plus Low-Rank) and DPLR - (Diagonal Plus Low-Rank) parameterizations. - - Args: - measure (str): The type of measure to use for initialization. Special - values include 'dplr' for Diagonal Plus Low-Rank and 'diag-*' for - various diagonal initializations. - N (int): State size. - R (int): Rank for the low-rank component (used in DPLR parameterization). - H (int): Number of independent SSM copies. - **ssm_args: Additional keyword arguments passed to the specific - initialization functions. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: - - w (torch.Tensor): Diagonal coefficients, shape (H, N). - - P (torch.Tensor): Low-rank term, shape (R, H, N). - - B (torch.Tensor): Input projection, shape (H, N). - - V (torch.Tensor): Transformation matrix, shape (H, N, N). - - Raises: - NotImplementedError: If an unsupported measure is specified. - - Note: - For 'dplr' and 'diag-*' measures, the function calls `dplr()`. - For other measures, it calls `nplr()`. - - Example: - w, P, B, V = ssm('legs', 64, 2, 4) - # Initializes SSM parameters for Legendre measure with - # state size 64, rank 2, and 4 independent copies + """Dispatcher to create single SSM initialization. + + N: state size + R: rank (for DPLR parameterization) + H: number of independent SSM copies """ if measure == "dplr": w, P, B, V = dplr(N=N, rank=R, H=H, **ssm_args) @@ -807,41 +586,6 @@ def ssm(measure, N, R, H, **ssm_args): def combination(measures, N, R, S, **ssm_args): - """ - Create a combination of multiple State Space Model (SSM) initializations. - - This function allows for the creation of SSM parameters using multiple measures, - effectively combining different types of SSMs into a single set of parameters. - - Args: - measures (str or list): Either a string specifying a predefined combination - ('hippo', 'diag', 'all') or a list of measure names. - N (int): State size for each SSM. - R (int): Rank for the low-rank component in each SSM. - S (int): Total number of independent trainable SSM copies. - **ssm_args: Additional keyword arguments passed to the ssm() function. - - Returns: - Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: A tuple containing: - - w (torch.Tensor): Combined diagonal coefficients, shape (S, N). - - P (torch.Tensor): Combined low-rank terms, shape (R, S, N). - - B (torch.Tensor): Combined input projections, shape (S, N). - - V (torch.Tensor): Combined transformation matrices, shape (S, N, N). - - Raises: - AssertionError: If S is not divisible by the number of measures. - - Note: - Predefined combinations: - - 'hippo': Combines 'legs' and 'fourier' measures. - - 'diag': Combines 'diag-inv' and 'diag-lin' measures. - - 'all': Combines 'legs', 'fourier', 'diag-inv', and 'diag-lin' measures. - - Example: - w, P, B, V = combination('hippo', 64, 2, 8) - # Creates a combination of Legendre and Fourier SSMs - # with state size 64, rank 2, and 8 total copies (4 of each type) - """ if isinstance(measures, str): measures = combinations[measures] if measures in combinations else [measures] @@ -860,42 +604,10 @@ def combination(measures, N, R, S, **ssm_args): class OptimModule(nn.Module): - """ - A module that allows registering buffers/parameters with configurable optimizer hyperparameters. - - This class extends nn.Module to provide a custom registration method for tensors, - allowing them to be registered as either buffers or parameters with specific - learning rates and weight decay settings. - - The main feature is the ability to set per-parameter learning rates, - which can be useful for fine-tuning specific parts of a model or - implementing more complex optimization strategies. - """ + """Interface for Module that allows registering buffers/parameters with configurable optimizer hyperparameters. # noqa""" def register(self, name, tensor, lr=None): - """ - Register a tensor as a buffer or parameter with configurable learning rate. - - This method allows for flexible registration of tensors within the module, - with the option to specify a custom learning rate for parameters. - - Args: - name (str): The name under which the tensor will be registered. - tensor (torch.Tensor): The tensor to be registered. - lr (float, optional): The learning rate for the tensor if registered as a parameter. - If None, uses the default learning rate. If 0.0, registers as a buffer. - - Note: - - If lr is 0.0, the tensor is registered as a buffer (non-learnable). - - If lr is not None and not 0.0, the tensor is registered as a parameter - with the specified learning rate and zero weight decay. - - The learning rate is stored in the '_optim' attribute of the parameter, - which can be used by custom optimizers. - - Example: - self.register('my_tensor', torch.randn(10), lr=0.01) - # Registers 'my_tensor' as a parameter with learning rate 0.01 - """ + """Register a tensor with a configurable learning rate and 0 weight decay.""" if lr == 0.0: self.register_buffer(name, tensor) else: @@ -908,18 +620,10 @@ def register(self, name, tensor, lr=None): class SSKernelNPLR(OptimModule): - """ - Stores a representation of and computes the SSKernel function K_L(A^dt, B^dt, C). + """Stores a representation of and computes the SSKernel function. - This class implements the Normal Plus Low-Rank (NPLR) parameterization of the - SSM (State Space Model) kernel. It provides methods to compute the kernel function - and its state space realization for the discretized state space model. - - The kernel is represented as K_L(A^dt, B^dt, C) where A is parameterized - as A = w - PP^*, w is complex diagonal, and P is low rank. - - This implementation includes various optimizations and caching mechanisms - to improve computational efficiency, especially for long sequence lengths. + K_L(A^dt, B^dt, C) corresponding to a discretized state space, + where A is Normal + Low Rank (NPLR) """ @torch.no_grad() @@ -1091,33 +795,15 @@ def _w(self): return w def forward(self, state=None, rate=1.0, L=None): - """ - Compute the SSKernel function for the given parameters. - - This method calculates the convolution kernel and optionally the output - from an initial state. - - Args: - state (torch.Tensor, optional): Initial state of shape (B, H, N). - If provided, the method also computes the output from this state. - rate (float, optional): Sampling rate factor. Defaults to 1.0. - L (int, optional): Target length of the kernel. If None, uses the - internal length self.L. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - The computed convolution kernel of shape (C, H, L). - - If state is provided, also returns the output from the initial - state of shape (B, H, L). - - Note: - - The method automatically handles length adjustment and caching. - - It supports various discretization methods specified in self.disc. - - The computation leverages FFT for efficiency in long sequence computations. - - Example: - kernel, _ = sskernel.forward(L=1024) - # Computes the kernel for a sequence of length 1024 + """Forward pass. + + state: (B, H, N) initial state + rate: sampling rate factor + L: target length + + returns: + (C, H, L) convolution kernel (generally C=1) + (B, H, L) output from initial state """ # Initialize C~ # if necessary (done in forward pass so it's on the correct device) @@ -1426,30 +1112,6 @@ def _setup_step(self, mode="dense"): ) def default_state(self, *batch_shape): - """ - Create a default initial state for the SSM. - - This method generates a zero-initialized state tensor with the appropriate - shape and data type for the SSM. - - Args: - *batch_shape: Variable length argument to specify the batch dimensions - of the state tensor. - - Returns: - torch.Tensor: A zero-initialized state tensor of shape (*batch_shape, H, N), - where H is the number of independent SSM copies and N is the state size. - - Note: - - The state tensor is created on the same device as the model parameters. - - The data type of the state matches that of the model's parameters. - - This method is useful for initializing the state when starting a new sequence. - - Example: - state = sskernel.default_state(32, 10) - # Creates a state tensor of shape (32, 10, H, N) for a batch of 32 sequences, - # each with 10 independent dimensions. - """ C = _r2c(self.C) N = C.size(-1) H = C.size(-2) @@ -1491,33 +1153,10 @@ def default_state(self, *batch_shape): return state def step(self, u, state): - """ - Perform a single step update of the SSM. - - This method applies one step of the state space model update, computing - the new state and output given an input and the current state. - - Args: - u (torch.Tensor): Input tensor of shape (B, H), where B is the batch size - and H is the number of independent SSM copies. - state (torch.Tensor): Current state tensor of shape (B, H, N), where N - is the state size. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - y (torch.Tensor): Output tensor of shape (B, C, H), where C is the - number of output channels. - - next_state (torch.Tensor): Updated state tensor of shape (B, H, N). - - Note: - - This method is intended for use during inference or validation. - - It assumes that self._setup_step() has been called to prepare the - necessary matrices for stepping. - - The computation uses the discretized state space matrices (dA, dB, dC). - - Example: - y, next_state = sskernel.step(u, state) - # Computes the output and next state for a single time step + """Step one time step as a recurrent model. + + Must have called self._setup_step() + and created state with self.default_state() before calling this """ if self._step_mode == "linear": new_state = self._step_state_linear(u, state) @@ -1528,21 +1167,7 @@ def step(self, u, state): class SSKernelDiag(OptimModule): - """ - Implements the SSKernel using a diagonal state matrix (S4D). - - This class represents a simplified version of the SSKernel where the state - matrix A is diagonal, leading to more efficient computations. It's part of - the S4D (Simplified State Space Sequence Model) architecture. - - The diagonal parameterization allows for faster forward and backward passes, - making it suitable for longer sequence modeling tasks. This implementation - supports various initialization schemes and discretization methods for the - state space model. - - The kernel is parameterized by diagonal matrices A and B, and a general C matrix, - allowing for efficient computation of the SSM kernel function. - """ + """Version using (complex) diagonal state matrix (S4D).""" def __init__( self, @@ -1621,34 +1246,15 @@ def _A(self): return A def forward(self, L, state=None, rate=1.0, u=None): - """ - Compute the SSKernel (S4D) function for the given parameters. - - This method calculates the convolution kernel and optionally the output - from an initial state for the diagonal SSM (S4D) model. - - Args: - L (int): Target length of the kernel. - state (torch.Tensor, optional): Initial state of shape (B, H, N). - If provided, the method also computes the output from this state. - rate (float, optional): Sampling rate factor. Defaults to 1.0. - u (torch.Tensor, optional): Input sequence. Not used in the current implementation. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - The computed convolution kernel of shape (C, H, L). - - If state is provided, also returns the output from the initial - state of shape (B, C, H, L). - - Note: - - The method supports different discretization methods (zoh, bilinear, dss) - specified in self.disc. - - It uses efficient computations leveraging the diagonal structure of A. - - The method handles rate adjustment and supports optional bandlimiting. - - Example: - kernel, _ = sskernel_diag.forward(L=1024, rate=0.5) - # Computes the kernel for a sequence of length 1024 at half the original rate + """Forward pass. + + state: (B, H, N) initial state + rate: sampling rate factor + L: target length + + returns: + (C, H, L) convolution kernel (generally C=1) + (B, H, L) output from initial state """ dt = torch.exp(self.log_dt) * rate # (H) C = _r2c(self.C) # (C H N) @@ -1738,31 +1344,6 @@ def _setup_step(self): ) # or * dtA / A def default_state(self, *batch_shape): - """ - Create a default initial state for the SSM with diagonal state matrix. - - This method generates a zero-initialized state tensor with the appropriate - shape and data type for the S4D (Simplified State Space Sequence) model. - - Args: - *batch_shape: Variable length argument to specify the batch dimensions - of the state tensor. - - Returns: - torch.Tensor: A zero-initialized state tensor of shape (*batch_shape, H, N), - where H is the number of independent SSM copies and N is the state size. - - Note: - - The state tensor is created on the same device as the model parameters. - - The data type of the state matches that of the model's C parameter. - - This method is useful for initializing the state when starting a new sequence - or batch of sequences. - - Example: - state = sskernel_diag.default_state(32, 10) - # Creates a state tensor of shape (32, 10, H, N) for a batch of 32 sequences, - # each with 10 independent dimensions. - """ C = _r2c(self.C) state = torch.zeros( *batch_shape, self.H, self.N, dtype=C.dtype, device=C.device @@ -1770,36 +1351,6 @@ def default_state(self, *batch_shape): return state def step(self, u, state): - """ - Perform a single step update of the SSM with diagonal state matrix. - - This method applies one step of the state space model update for the S4D model, - computing the new state and output given an input and the current state. - - Args: - u (torch.Tensor): Input tensor of shape (B, H), where B is the batch size - and H is the number of independent SSM copies. - state (torch.Tensor): Current state tensor of shape (B, H, N), where N - is the state size. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - y (torch.Tensor): Output tensor of shape (B, C, H), where C is the - number of output channels. - - next_state (torch.Tensor): Updated state tensor of shape (B, H, N). - - Note: - - This method is optimized for the diagonal structure of the state matrix. - - It assumes that self._setup_step() has been called to prepare the - necessary matrices for stepping. - - The computation uses the discretized state space matrices (dA, dB, dC). - - The output is real-valued, obtained by taking twice the real part of - the complex computation. - - Example: - y, next_state = sskernel_diag.step(u, state) - # Computes the output and next state for a single time step in the S4D model - """ next_state = contract("h n, b h n -> b h n", self.dA, state) + contract( "h n, b h -> b h n", self.dB, u ) @@ -1807,33 +1358,7 @@ def step(self, u, state): return 2 * y.real, next_state def forward_state(self, u, state): - """ - Compute the next state of the SSM given an input sequence. - - This method calculates the final state after processing an entire input sequence - using the S4D (Simplified State Space Sequence) model with a diagonal state matrix. - - Args: - u (torch.Tensor): Input sequence tensor of shape (B, H, L), where B is the - batch size, H is the number of independent SSM copies, - and L is the sequence length. - state (torch.Tensor): Initial state tensor of shape (B, H, N), where N - is the state size. - - Returns: - torch.Tensor: The final state after processing the entire input sequence, - with shape (B, H, N). - - Note: - - This method is more efficient than repeatedly calling step() for each - time step, as it computes the state transition for the entire sequence at once. - - It uses the log_vandermonde_transpose function for efficient computation. - - The method automatically sets up the step matrices if not already done. - - Example: - final_state = sskernel_diag.forward_state(input_sequence, initial_state) - # Computes the final state after processing the entire input_sequence - """ + self._setup_step() AL = self.dA ** u.size(-1) u = u.flip(-1).to(self.dA).contiguous() # (B H L) v = log_vandermonde_transpose(u, self.dB, self.dA.log(), u.size(-1)) @@ -1842,18 +1367,13 @@ def forward_state(self, u, state): class SSKernel(nn.Module): - """ - Wrapper class for different State Space Kernel parameterizations. - - This class serves as a unified interface for various implementations of - State Space Kernels, including NPLR (Normal Plus Low-Rank) and Diagonal - parameterizations. It provides a common API for initializing and using - different SSM (State Space Model) kernels, making it easier to experiment - with and switch between different kernel types. + """Wrapper around SSKernel parameterizations. - The SSKernel supports various initialization schemes, measures, and - configuration options, allowing for flexible and powerful sequence modeling. - It can be used as a core component in S4 (Structured State Space Sequence) models. + The SSKernel is expected to support the interface + forward() + default_state() + _setup_step() + step() """ def __init__( @@ -1990,66 +1510,18 @@ def __init__( raise NotImplementedError(f"mode={mode} is not valid") def forward(self, state=None, L=None, rate=None): - """ - Compute the SSKernel function for the given parameters. - - This method is a wrapper around the underlying kernel's forward method, - providing a unified interface for different kernel implementations. - - Args: - state (torch.Tensor, optional): Initial state. Shape depends on the - specific kernel implementation. - L (int, optional): Target length of the kernel. If None, uses the - default length specified in the kernel. - rate (float, optional): Sampling rate factor. Defaults to None. - - Returns: - The return type and shape depend on the specific kernel implementation, - but typically includes: - - The computed convolution kernel - - Optionally, the output from the initial state if provided - - Note: - This method directly calls the forward method of the underlying - kernel (either SSKernelNPLR or SSKernelDiag), passing through all - provided arguments. - - Example: - kernel, state_output = sskernel(state=initial_state, L=1024, rate=0.5) - # Computes the kernel and state output for a sequence of length 1024 - # at half the original rate - """ return self.kernel(state=state, L=L, rate=rate) @torch.no_grad() def forward_state(self, u, state): - """ - Compute the next state of the SSM given an input sequence. - - This method forwards the state through a sequence using the underlying - kernel's state transition logic. - - Args: - u (torch.Tensor): Input sequence tensor. The shape should be - compatible with the underlying kernel, typically (B, H, L), - where B is batch size, H is the number of independent SSM copies, - and L is the sequence length. - state (torch.Tensor): Initial state tensor. The shape should match - the state representation of the underlying kernel. - - Returns: - torch.Tensor: The final state after processing the entire input sequence. - The shape will match the input state shape. - - Note: - - This method is particularly useful for tracking the state evolution - over a sequence without computing the full convolution output. - - If the underlying kernel doesn't have a `forward_state` method, - this function falls back to a manual state update using `_setup_state`. - - Example: - final_state = sskernel.forward_state(input_sequence, initial_state) - # Computes the final state after processing the entire input_sequence + """Forward the state through a sequence. + + i.e. computes the state after passing chunk through SSM + + state: (B, H, N) + u: (B, H, L) + + Returns: (B, H, N) """ if hasattr(self.kernel, "forward_state"): return self.kernel.forward_state(u, state) @@ -2081,91 +1553,14 @@ def _setup_step(self, **kwargs): self.kernel._setup_step(**kwargs) def step(self, u, state, **kwargs): - """ - Perform a single step update of the SSM. - - This method applies one step of the state space model update, computing - the new state and output given an input and the current state. It delegates - the computation to the underlying kernel's step method. - - Args: - u (torch.Tensor): Input tensor for the current step. The shape should be - compatible with the underlying kernel, typically (B, H) where B is - the batch size and H is the number of independent SSM copies. - state (torch.Tensor): Current state tensor. The shape should match the - state representation of the underlying kernel. - **kwargs: Additional keyword arguments that will be passed to the - underlying kernel's step method. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - y (torch.Tensor): Output tensor for the current step. - - next_state (torch.Tensor): Updated state tensor. - - Note: - - This method is intended for use during inference or validation, allowing - for step-by-step processing of a sequence. - - It assumes that the underlying kernel has been properly initialized and - its `_setup_step` method has been called if necessary. - - Example: - output, new_state = sskernel.step(input, current_state) - # Computes the output and next state for a single time step - """ y, state = self.kernel.step(u, state, **kwargs) return y, state def default_state(self, *args, **kwargs): - """ - Create a default initial state for the SSM. - - This method generates a default state tensor appropriate for the underlying - kernel implementation. It serves as a convenience wrapper around the - kernel's own default_state method. - - Args: - *args: Variable length argument list that will be passed to the - underlying kernel's default_state method. - **kwargs: Arbitrary keyword arguments that will be passed to the - underlying kernel's default_state method. - - Returns: - torch.Tensor: A default state tensor. The shape and content of this - tensor depend on the specific implementation of the - underlying kernel. - - Note: - - The returned state is typically a zero-initialized tensor with - appropriate dimensions for the SSM. - - This method is useful for initializing the state when starting - a new sequence or when no prior state information is available. - - Example: - initial_state = sskernel.default_state(32, 10) - # Creates a default state tensor for a batch of 32 sequences, - # potentially with 10 independent dimensions (depending on the kernel) - """ return self.kernel.default_state(*args, **kwargs) class S4(nn.Module): - """ - Structured State Space Sequence (S4) model. - - This class implements the S4 model, which combines a State Space Model (SSM) - with position-wise feedforward layers. S4 is designed for efficient and - effective modeling of long sequences. - - The S4 module includes: - - An SSM kernel for sequence modeling - - Optional gating and bottleneck mechanisms - - Position-wise feedforward layers with various activation functions - - Support for bidirectional processing - - The model can be configured with different SSM kernels (e.g., NPLR or diagonal) - and allows for extensive customization of its components and hyperparameters. - """ - def __init__( self, d_model, @@ -2296,35 +1691,12 @@ def __init__( ) def forward(self, u, state=None, rate=1.0, lengths=None, **kwargs): - """ - Forward pass of the S4 model. - - This method applies the S4 transformation to the input sequence, including - the SSM convolution and feedforward layers. - - Args: - u (torch.Tensor): Input tensor of shape (B, H, L) if self.transposed else (B, L, H), - where B is batch size, H is hidden size, and L is sequence length. - state (torch.Tensor, optional): Initial state for the SSM. Shape depends on the SSM configuration. - rate (float, optional): Sampling rate factor for the SSM. Defaults to 1.0. - lengths (torch.Tensor or int, optional): Actual lengths of sequences in the batch. - Used for masking padded elements. - **kwargs: Additional keyword arguments passed to the SSM kernel. - - Returns: - Tuple[torch.Tensor, Optional[torch.Tensor]]: - - Output tensor of the same shape as the input. - - Next state of the SSM if state argument was provided, else None. - - Note: - - The method handles different axis orderings based on self.transposed. - - It applies sequence masking if lengths are provided. - - The SSM convolution is combined with skip connections and feedforward layers. - - Supports optional gating and bottleneck mechanisms if configured. - - Example: - output, next_state = s4_model(input_sequence, state=initial_state, rate=0.5) - # Processes the input_sequence and returns the output along with the next state + """Forward pass. + + u: (B H L) if self.transposed else (B L H) + state: (H N) never needed unless you know what you're doing + + Returns: same shape as u """ if not self.transposed: u = u.transpose(-1, -2) @@ -2407,58 +1779,16 @@ def forward(self, u, state=None, rate=1.0, lengths=None, **kwargs): return y, next_state def setup_step(self, **kwargs): - """ - Set up the S4 model for step-by-step inference. - - This method prepares the S4 model for sequential processing, typically used - during inference or generation tasks. It initializes the underlying SSM kernel - for step-wise computation. - - Args: - **kwargs: Arbitrary keyword arguments that will be passed to the - SSM kernel's _setup_step method. - - Note: - - This method should be called before using the step() method for - sequential processing. - - It configures the internal state of the SSM kernel for efficient - step-by-step computation. - - The exact behavior depends on the specific SSM kernel implementation. - - Example: - s4_model.setup_step() - # Prepares the model for subsequent calls to s4_model.step() - """ + self.kernel._setup_step(**kwargs) def step(self, u, state, **kwargs): - """ - Perform a single step update of the S4 model. - - This method processes a single time step input, updating the model's state - and producing an output. It's designed for use in sequential processing - scenarios, typically during inference or generation. - - Args: - u (torch.Tensor): Input tensor for the current step. Shape should be (B, H) - where B is the batch size and H is the hidden size. - state (torch.Tensor): Current state of the model. The shape depends on - the specific SSM kernel configuration. - **kwargs: Additional keyword arguments passed to the SSM kernel's step method. - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - - y (torch.Tensor): Output tensor for the current step, shape (B, H). - - next_state (torch.Tensor): Updated state tensor for use in the next step. - - Note: - - This method assumes that setup_step() has been called beforehand. - - It applies the SSM kernel step, followed by activation and the output linear layer. - - The method is not intended for training, but for inference or generation. - - If configured, it applies gating mechanism to the output. - - Example: - output, new_state = s4_model.step(input_step, current_state) - # Processes a single time step and returns the output and updated state + """Step one time step as a recurrent model. + + Intended to be used during validation. + + u: (B H) + state: (B H N) + Returns: output (B H), state (B H N) """ assert not self.training y, next_state = self.kernel.step(u, state) # (B C H) @@ -2472,57 +1802,10 @@ def step(self, u, state, **kwargs): return y, next_state def default_state(self, *batch_shape, device=None): - """ - Create a default initial state for the S4 model. - - This method generates a default state tensor appropriate for the S4 model's - SSM kernel. It's typically used to initialize the state at the beginning of - a sequence processing task. - - Args: - *batch_shape: Variable length argument list specifying the batch dimensions - of the state tensor. - device (torch.device, optional): The device on which to create the state tensor. - If None, uses the device of the model's parameters. - - Returns: - torch.Tensor: A default state tensor. The shape and content of this tensor - depend on the specific implementation of the underlying SSM kernel. - - Note: - - The returned state is typically a zero-initialized tensor with dimensions - that match the SSM kernel's requirements. - - This method is useful when starting sequence processing without any prior state, - or when resetting the model's state. - - The device of the returned tensor will match the model's parameters if not specified. - - Example: - initial_state = s4_model.default_state(32) - # Creates a default state tensor for a batch of 32 sequences - """ # kernel is not a SequenceModule so it doesn't need to adhere to same interface # the kernel will know the device of its own parameters return self.kernel.default_state(*batch_shape) @property def d_output(self): - """ - Get the output dimension of the S4 model. - - This property returns the dimension of the output produced by the S4 model - for each time step. - - Returns: - int: The output dimension, which is equal to the model's hidden dimension (d_model). - - Note: - - This is typically used to determine the size of the model's output - for downstream tasks or for connecting to other layers. - - The output dimension is the same as the input dimension (d_model) in the - standard S4 configuration, maintaining the sequence's channel size. - - Example: - output_size = s4_model.d_output - # Retrieves the output dimension of the S4 model - """ return self.d_model diff --git a/espnet2/asr/state_spaces/utils.py b/espnet2/asr/state_spaces/utils.py index af8d9a1d2d9..7234cf07888 100644 --- a/espnet2/asr/state_spaces/utils.py +++ b/espnet2/asr/state_spaces/utils.py @@ -9,85 +9,17 @@ def is_list(x): - """ - Check if the input is a list-like sequence (excluding strings). - - This function determines whether the input is a sequence (like a list or tuple) - but not a string. - - Args: - x: The object to be checked. - - Returns: - bool: True if the input is a list-like sequence, False otherwise. - - Examples: - >>> is_list([1, 2, 3]) - True - >>> is_list((1, 2, 3)) - True - >>> is_list("abc") - False - >>> is_list(123) - False - """ return isinstance(x, Sequence) and not isinstance(x, str) def is_dict(x): - """ - Check if the input is a dictionary-like object. - - This function determines whether the input is an instance of a Mapping, - which includes dictionaries and other mapping types. - - Args: - x: The object to be checked. - - Returns: - bool: True if the input is a dictionary-like object, False otherwise. - - Examples: - >>> is_dict({'a': 1, 'b': 2}) - True - >>> is_dict(dict(a=1, b=2)) - True - >>> is_dict([1, 2, 3]) - False - >>> is_dict("abc") - False - """ return isinstance(x, Mapping) def to_dict(x, recursive=True): - """ - Convert a Sequence or Mapping object to a dictionary. + """Convert Sequence or Mapping object to dict. - This function converts list-like objects to dictionaries with integer keys, - and recursively converts nested structures if specified. - - Args: - x: The object to be converted to a dictionary. - recursive (bool, optional): If True, recursively convert nested structures. - Defaults to True. - - Returns: - dict: The converted dictionary. - - Examples: - >>> to_dict([1, 2, 3]) - {0: 1, 1: 2, 2: 3} - >>> to_dict({'a': [1, 2], 'b': {'c': 3}}, recursive=True) - {'a': {0: 1, 1: 2}, 'b': {'c': 3}} - >>> to_dict({'a': [1, 2], 'b': {'c': 3}}, recursive=False) - {'a': [1, 2], 'b': {'c': 3}} - >>> to_dict(5) - 5 - - Note: - - Lists are converted to dictionaries with integer keys. - - If the input is neither a Sequence nor a Mapping, it is returned unchanged. + lists get converted to {0: x[0], 1: x[1], ...} """ if is_list(x): x = {i: v for i, v in enumerate(x)} @@ -101,38 +33,11 @@ def to_dict(x, recursive=True): def to_list(x, recursive=False): - """ - Convert an object to a list. - - This function converts Sequence objects (except strings) to lists and handles - recursive conversion of nested structures if specified. - - Args: - x: The object to be converted to a list. - recursive (bool, optional): If True, recursively convert nested structures. - Defaults to False. + """Convert an object to list. - Returns: - list: The converted list. + If Sequence (e.g. list, tuple, Listconfig): just return it - Examples: - >>> to_list((1, 2, 3)) - [1, 2, 3] - >>> to_list([1, 2, 3]) - [1, 2, 3] - >>> to_list("abc") - ["abc"] - >>> to_list(5) - [5] - >>> to_list([1, [2, 3], 4], recursive=True) - [1, [2, 3], 4] - >>> to_list([1, [2, 3], 4], recursive=False) - [1, [2, 3], 4] - - Note: - - If the input is not a Sequence and recursive is False, it will be wrapped in a list. - - If recursive is True and the input is not a Sequence, it will be returned unchanged. - - Strings are treated as non-Sequence objects and will be wrapped in a list if recursive is False. + Special case: If non-recursive and not a list, wrap in list """ if is_list(x): if recursive: @@ -147,38 +52,6 @@ def to_list(x, recursive=False): def extract_attrs_from_obj(obj, *attrs): - """ - Extract specified attributes from an object. - - This function retrieves the values of specified attributes from the given object. - If the object is None, it returns an empty list. - - Args: - obj: The object from which to extract attributes. - *attrs: Variable length argument list of attribute names to extract. - - Returns: - list: A list containing the values of the specified attributes. - If an attribute doesn't exist, None is used as its value. - - Raises: - AssertionError: If obj is None and attrs is not empty. - - Examples: - >>> class Example: - ... def __init__(self): - ... self.a = 1 - ... self.b = "test" - >>> obj = Example() - >>> extract_attrs_from_obj(obj, "a", "b", "c") - [1, "test", None] - >>> extract_attrs_from_obj(None) - [] - - Note: - - If obj is None, the function expects no attributes to be specified. - - For non-existent attributes, None is used as the value in the returned list. - """ if obj is None: assert len(attrs) == 0 return [] @@ -186,43 +59,15 @@ def extract_attrs_from_obj(obj, *attrs): def instantiate(registry, config, *args, partial=False, wrap=None, **kwargs): - """ - Instantiate a registered module based on the provided configuration. - - This function creates an instance of a registered module using the provided - configuration and additional arguments. It supports various instantiation - scenarios and can optionally wrap the target class. - - Args: - registry (dict): A dictionary mapping names to functions or target paths. - config (dict or str): Configuration for instantiation. If a string, it's - treated as the key for the registry. If a dict, it should contain a - '_name_' key indicating which element of the registry to use. - *args: Additional positional arguments to pass to the constructor. - partial (bool, optional): If True, returns a partial function instead of - an instantiated object. Defaults to False. - wrap (callable, optional): A function to wrap the target class. Defaults to None. - **kwargs: Additional keyword arguments to override the config and pass - to the target constructor. - - Returns: - object or functools.partial: The instantiated object or a partial function, - depending on the 'partial' parameter. + """Instantiate registered module. - Raises: - NotImplementedError: If the instantiate target is neither a string nor callable. - - Examples: - >>> registry = {'model': 'models.SequenceModel'} - >>> config = {'_name_': 'model', 'hidden_size': 128} - >>> model = instantiate(registry, config) - >>> partial_model = instantiate(registry, config, partial=True) - >>> wrapped_model = instantiate(registry, config, wrap=some_wrapper_function) - - Note: - - The function supports both string-based and callable-based registry entries. - - If 'config' is a string, it's used as the key for the registry. - - The '_name_' key is restored in the config after instantiation. + registry: Dictionary mapping names to functions or target paths + (e.g. {'model': 'models.SequenceModel'}) + config: Dictionary with a '_name_' key indicating which element of the registry + to grab, and kwargs to be passed into the target constructor + wrap: wrap the target class (e.g. ema optimizer or tasks.wrap) + *args, **kwargs: additional arguments + to override the config to pass into the target constructor """ # Case 1: no config if config is None: @@ -261,74 +106,12 @@ def instantiate(registry, config, *args, partial=False, wrap=None, **kwargs): def get_class(registry, _name_): - """ - Retrieve a class from the registry based on the provided name. - - This function uses Hydra's get_class utility to fetch the class specified - by the _name_ parameter from the given registry. - - Args: - registry (dict): A dictionary mapping names to class paths. - _name_ (str): The name of the class to retrieve from the registry. - - Returns: - type: The class object corresponding to the specified name. - - Raises: - Any exceptions raised by hydra.utils.get_class. - - Examples: - >>> registry = {'MyClass': 'path.to.MyClass'} - >>> MyClass = get_class(registry, 'MyClass') - >>> instance = MyClass() - - Note: - - This function includes a breakpoint() call, which will pause execution - when the function is called. This is likely for debugging purposes and - should be removed in production code. - - The function relies on Hydra's get_class utility, which dynamically - imports and returns the specified class. - """ + breakpoint() return hydra.utils.get_class(path=registry[_name_]) def omegaconf_filter_keys(d, fn=None): - """ - Filter keys in an OmegaConf structure based on a given function. - - This function traverses through a nested OmegaConf structure (DictConfig or ListConfig) - and filters keys based on the provided function. It supports recursive filtering - for nested structures. - - Args: - d (Union[ListConfig, DictConfig, Any]): The OmegaConf structure to filter. - fn (Callable[[str], bool], optional): A function that takes a key as input - and returns True if the key should be kept, False otherwise. - If None, all keys are kept. Defaults to None. - - Returns: - Union[ListConfig, DictConfig, Any]: A new OmegaConf structure with filtered keys. - - Examples: - >>> from omegaconf import DictConfig, ListConfig - >>> config = DictConfig({'a': 1, 'b': ListConfig([1, 2]), 'c': DictConfig({'d': 3, 'e': 4})}) - >>> filtered = omegaconf_filter_keys(config, lambda k: k != 'b') - >>> print(filtered) - {'a': 1, 'c': {'d': 3, 'e': 4}} - - >>> def filter_func(key): - ... return not key.startswith('_') - >>> config = DictConfig({'a': 1, '_b': 2, 'c': DictConfig({'d': 3, '_e': 4})}) - >>> filtered = omegaconf_filter_keys(config, filter_func) - >>> print(filtered) - {'a': 1, 'c': {'d': 3}} - - Note: - - If no filter function is provided, all keys are kept. - - The function preserves the OmegaConf structure (DictConfig for dictionaries, - ListConfig for lists) in the returned object. - - For non-dict and non-list inputs, the original input is returned unchanged. - """ + """Only keep keys where fn(key) is True. Support nested DictConfig.""" if fn is None: def fn(_): diff --git a/espnet2/asr/transducer/error_calculator.py b/espnet2/asr/transducer/error_calculator.py index e35ad2aab40..4ddf9cc9b7b 100644 --- a/espnet2/asr/transducer/error_calculator.py +++ b/espnet2/asr/transducer/error_calculator.py @@ -9,42 +9,16 @@ class ErrorCalculatorTransducer(object): - """ - Error Calculator for Transducer-based models in speech recognition. - - This class calculates Character Error Rate (CER) and Word Error Rate (WER) for - transducer models used in automatic speech recognition tasks. - - Attributes: - beam_search (BeamSearchTransducer): Beam search decoder for transducer models. - decoder (AbsDecoder): Decoder module. - token_list (List[int]): List of token IDs. - space (str): Space symbol. - blank (str): Blank symbol. - report_cer (bool): Flag to compute CER. - report_wer (bool): Flag to compute WER. + """Calculate CER and WER for transducer models. Args: - decoder (AbsDecoder): Decoder module. - joint_network (torch.nn.Module): Joint network module. - token_list (List[int]): List of token IDs. - sym_space (str): Space symbol. - sym_blank (str): Blank symbol. - report_cer (bool, optional): Whether to compute CER. Defaults to False. - report_wer (bool, optional): Whether to compute WER. Defaults to False. - - Example: - >>> decoder = TransducerDecoder(...) - >>> joint_network = JointNetwork(...) - >>> token_list = ["", "a", "b", "c", ...] - >>> error_calc = ErrorCalculatorTransducer( - ... decoder, joint_network, token_list, sym_space="", sym_blank="" - ... ) - >>> cer, wer = error_calc(encoder_out, target) - - Note: - This class uses beam search decoding to generate predictions and then - calculates the error rates by comparing them with the target sequences. + decoder: Decoder module. + token_list: List of tokens. + sym_space: Space symbol. + sym_blank: Blank symbol. + report_cer: Whether to compute CER. + report_wer: Whether to compute WER. + """ def __init__( @@ -78,38 +52,16 @@ def __init__( self.report_wer = report_wer def __call__(self, encoder_out: torch.Tensor, target: torch.Tensor): - """ - Calculate sentence-level WER/CER score for Transducer model. - - This method performs beam search decoding on the encoder output and calculates - the Character Error Rate (CER) and Word Error Rate (WER) by comparing the - decoded sequences with the target sequences. + """Calculate sentence-level WER/CER score for Transducer model. Args: - encoder_out (torch.Tensor): Encoder output sequences. Shape: (B, T, D_enc), - where B is the batch size, T is the sequence length, and D_enc is the - encoder output dimension. - target (torch.Tensor): Target label ID sequences. Shape: (B, L), where B is - the batch size and L is the target sequence length. + encoder_out: Encoder output sequences. (B, T, D_enc) + target: Target label ID sequences. (B, L) Returns: - tuple: A tuple containing: - - cer (float or None): Sentence-level Character Error Rate if report_cer - is True, else None. - - wer (float or None): Sentence-level Word Error Rate if report_wer is - True, else None. - - Example: - >>> encoder_out = torch.randn(2, 100, 256) # Batch size 2, seq length 100 - >>> target = torch.randint(0, 1000, (2, 50)) # Batch size 2, target length 50 - >>> error_calc = ErrorCalculatorTransducer(...) - >>> cer, wer = error_calc(encoder_out, target) - >>> print(f"CER: {cer}, WER: {wer}") - - Note: - This method uses the beam search algorithm to decode the encoder output. - The resulting predictions are then converted to character sequences for - error rate calculation. + : Sentence-level CER score. + : Sentence-level WER score. + """ cer, wer = None, None @@ -137,38 +89,16 @@ def __call__(self, encoder_out: torch.Tensor, target: torch.Tensor): def convert_to_char( self, pred: torch.Tensor, target: torch.Tensor ) -> Tuple[List, List]: - """ - Convert label ID sequences to character sequences. - - This method transforms the predicted and target label ID sequences into - character sequences by mapping each ID to its corresponding token. It also - handles the replacement of space and blank symbols. + """Convert label ID sequences to character sequences. Args: - pred (torch.Tensor): Prediction label ID sequences. Shape: (B, U), where B - is the batch size and U is the length of the predicted sequence. - target (torch.Tensor): Target label ID sequences. Shape: (B, L), where B is - the batch size and L is the length of the target sequence. + pred: Prediction label ID sequences. (B, U) + target: Target label ID sequences. (B, L) Returns: - tuple: A tuple containing two lists: - - char_pred (List[str]): List of predicted character sequences, one for - each sample in the batch. - - char_target (List[str]): List of target character sequences, one for - each sample in the batch. - - Example: - >>> pred = torch.tensor([[1, 2, 3], [4, 5, 6]]) - >>> target = torch.tensor([[1, 2, 3], [4, 5, 6]]) - >>> error_calc = ErrorCalculatorTransducer(...) - >>> char_pred, char_target = error_calc.convert_to_char(pred, target) - >>> print(f"Predictions: {char_pred}") - >>> print(f"Targets: {char_target}") - - Note: - This method replaces the space symbol with an actual space character and - removes any occurrence of the blank symbol in both predicted and target - sequences. + char_pred: Prediction character sequences. (B, ?) + char_target: Target character sequences. (B, ?) + """ char_pred, char_target = [], [] @@ -190,35 +120,15 @@ def convert_to_char( def calculate_cer( self, char_pred: torch.Tensor, char_target: torch.Tensor ) -> float: - """ - Calculate sentence-level Character Error Rate (CER) score. - - This method computes the CER by comparing the predicted character sequences - with the target character sequences. It uses the Levenshtein distance - (edit distance) to measure the difference between sequences. + """Calculate sentence-level CER score. Args: - char_pred (List[str]): List of predicted character sequences, one for - each sample in the batch. - char_target (List[str]): List of target character sequences, one for - each sample in the batch. + char_pred: Prediction character sequences. (B, ?) + char_target: Target character sequences. (B, ?) Returns: - float: Average sentence-level CER score across the batch. - - Example: - >>> char_pred = ["hello wrld", "how r u"] - >>> char_target = ["hello world", "how are you"] - >>> error_calc = ErrorCalculatorTransducer(...) - >>> cer = error_calc.calculate_cer(char_pred, char_target) - >>> print(f"Character Error Rate: {cer}") - - Note: - - This method removes all space characters from both predicted and target - sequences before calculating the edit distance. - - The CER is calculated as the sum of edit distances divided by the sum of - target sequence lengths. - - This method requires the 'editdistance' package to be installed. + : Average sentence-level CER score. + """ import editdistance @@ -236,35 +146,15 @@ def calculate_cer( def calculate_wer( self, char_pred: torch.Tensor, char_target: torch.Tensor ) -> float: - """ - Calculate sentence-level Word Error Rate (WER) score. - - This method computes the WER by comparing the predicted word sequences - with the target word sequences. It uses the Levenshtein distance - (edit distance) to measure the difference between word sequences. + """Calculate sentence-level WER score. Args: - char_pred (List[str]): List of predicted character sequences, one for - each sample in the batch. - char_target (List[str]): List of target character sequences, one for - each sample in the batch. + char_pred: Prediction character sequences. (B, ?) + char_target: Target character sequences. (B, ?) Returns: - float: Average sentence-level WER score across the batch. - - Example: - >>> char_pred = ["hello world how are you", "this is a test"] - >>> char_target = ["hello world how are you doing", "this is only a test"] - >>> error_calc = ErrorCalculatorTransducer(...) - >>> wer = error_calc.calculate_wer(char_pred, char_target) - >>> print(f"Word Error Rate: {wer}") - - Note: - - This method splits each character sequence into words before calculating - the edit distance. - - The WER is calculated as the sum of edit distances divided by the sum of - target word sequence lengths. - - This method requires the 'editdistance' package to be installed. + : Average sentence-level WER score + """ import editdistance diff --git a/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py b/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py index aabf70f76b1..c007fa5536d 100644 --- a/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py +++ b/espnet2/asr/transducer/rnnt_multi_blank/rnnt.py @@ -48,44 +48,25 @@ def rnnt_loss_cpu( clamp: float, num_threads: int, ): - """ - Computes the RNN-T loss on CPU. + """Wrapper method for accessing CPU RNNT loss. - This function is a wrapper for the CPU implementation of the RNN-T loss, - ported from the warp-transducer project. + CPU implementation ported from [HawkAaron/warp-transducer] + (https://github.com/HawkAaron/warp-transducer). Args: acts: Activation tensor of shape [B, T, U, V+1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. - label_lengths: Lengths of the target sequences as a vector of ints [B]. - costs: Zero vector of length [B] where the computed costs will be stored. - grads: Zero tensor of shape [B, T, U, V+1] where the computed gradients will be stored. + input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. + label_lengths: Lengths of the target sequence as a vector of ints [B]. + costs: Zero vector of length [B] in which costs will be set. + grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. blank_label: Index of the blank token in the vocabulary. - fastemit_lambda: Float scaling factor for FastEmit regularization. - clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. - - Returns: - bool: True if the computation was successful. - - Raises: - RuntimeError: If there's an error in calculating the forward scores or workspace memory. - - Note: - B: batch size, T: input sequence length, U: output sequence length, V: vocabulary size - - Example: - >>> acts = torch.randn(2, 10, 5, 21) - >>> labels = torch.randint(0, 20, (2, 5)) - >>> input_lengths = torch.tensor([10, 8]) - >>> label_lengths = torch.tensor([5, 4]) - >>> costs = torch.zeros(2) - >>> grads = torch.zeros_like(acts) - >>> success = rnnt_loss_cpu(acts, labels, input_lengths, label_lengths, costs, grads, - ... blank_label=0, fastemit_lambda=0.0, clamp=0.0, num_threads=4) - >>> print(success) - True + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. + clamp: Float value. When set to value >= 0.0, will clamp the + gradient to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. """ # aliases @@ -175,44 +156,25 @@ def rnnt_loss_gpu( clamp: float, num_threads: int, ): - """ - Computes the RNN-T loss on GPU. + """Wrapper method for accessing GPU RNNT loss. - This function is a wrapper for the CUDA implementation of the RNN-T loss, - ported from the warp-transducer project. + CUDA implementation ported from [HawkAaron/warp-transducer] + (https://github.com/HawkAaron/warp-transducer). Args: acts: Activation tensor of shape [B, T, U, V+1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. - label_lengths: Lengths of the target sequences as a vector of ints [B]. - costs: Zero vector of length [B] where the computed costs will be stored. - grads: Zero tensor of shape [B, T, U, V+1] where the computed gradients will be stored. + input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. + label_lengths: Lengths of the target sequence as a vector of ints [B]. + costs: Zero vector of length [B] in which costs will be set. + grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. blank_label: Index of the blank token in the vocabulary. - fastemit_lambda: Float scaling factor for FastEmit regularization. - clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. - - Returns: - bool: True if the computation was successful. - - Raises: - RuntimeError: If there's an error in calculating the forward scores or workspace memory. - - Note: - B: batch size, T: input sequence length, U: output sequence length, V: vocabulary size - - Example: - >>> acts = torch.randn(2, 10, 5, 21, device='cuda') - >>> labels = torch.randint(0, 20, (2, 5), device='cuda') - >>> input_lengths = torch.tensor([10, 8], device='cuda') - >>> label_lengths = torch.tensor([5, 4], device='cuda') - >>> costs = torch.zeros(2, device='cuda') - >>> grads = torch.zeros_like(acts) - >>> success = rnnt_loss_gpu(acts, labels, input_lengths, label_lengths, costs, grads, - ... blank_label=0, fastemit_lambda=0.0, clamp=0.0, num_threads=4) - >>> print(success) - True + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. + clamp: Float value. When set to value >= 0.0, will clamp the + gradient to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. """ minibatch_size = acts.shape[0] @@ -308,50 +270,35 @@ def multiblank_rnnt_loss_gpu( num_threads: int, sigma: float, ): - """ - Computes the Multi-blank RNN-T loss on GPU. + """Wrapper method for accessing GPU Multi-blank RNNT loss - This function is a wrapper for the CUDA implementation of the Multi-blank RNN-T loss, - as described in the paper "Multi-blank Transducers for Speech Recognition" (https://arxiv.org/pdf/2211.03541.pdf). + CUDA implementation ported from [HawkAaron/warp-transducer] + (https://github.com/HawkAaron/warp-transducer). + Args: acts: Activation tensor of shape [B, T, U, V + num_big_blanks + 1]. labels: Ground truth labels of shape [B, U]. - input_lengths: Lengths of the acoustic sequences as a vector of ints [B]. - label_lengths: Lengths of the target sequences as a vector of ints [B]. - costs: Zero vector of length [B] where the computed costs will be stored. - grads: Zero tensor of shape [B, T, U, V + num_big_blanks + 1] where the computed gradients will be stored. + input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. + label_lengths: Lengths of the target sequence as a vector of ints [B]. + costs: Zero vector of length [B] in which costs will be set. + grads: Zero tensor of shape [B, T, U, V + num_big_blanks + 1] + where the gradient will be set. blank_label: Index of the standard blank token in the vocabulary. - big_blank_durations: List of supported durations for big blank symbols, e.g. [2, 4, 8]. - fastemit_lambda: Float scaling factor for FastEmit regularization. - clamp: Float value. When set to a value >= 0.0, the gradient will be clamped to [-clamp, clamp]. - num_threads: Number of threads for OpenMP. If negative, uses all available CPU cores. - sigma: Logit-undernormalization weight used in the multi-blank model. - - Returns: - bool: True if the computation was successful. - - Raises: - RuntimeError: If there's an error in calculating the forward scores or workspace memory. - - Note: - B: batch size, T: input sequence length, U: output sequence length, - V: vocabulary size, num_big_blanks: number of big blank symbols - - Example: - >>> acts = torch.randn(2, 10, 5, 24, device='cuda') # Assuming 3 big blanks - >>> labels = torch.randint(0, 20, (2, 5), device='cuda') - >>> input_lengths = torch.tensor([10, 8], device='cuda') - >>> label_lengths = torch.tensor([5, 4], device='cuda') - >>> costs = torch.zeros(2, device='cuda') - >>> grads = torch.zeros_like(acts) - >>> big_blank_durations = [2, 4, 8] - >>> success = multiblank_rnnt_loss_gpu(acts, labels, input_lengths, label_lengths, costs, grads, - ... blank_label=0, big_blank_durations=big_blank_durations, - ... fastemit_lambda=0.0, clamp=0.0, num_threads=4, sigma=0.5) - >>> print(success) - True + big_blank_durations: A list of supported durations for big blank symbols + in the model, e.g. [2, 4, 8]. Note we only include durations for ``big + blanks'' here and it should not include 1 for the standard blank. + Those big blanks have vocabulary indices after the standard blank index. + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. + clamp: Float value. When set to value >= 0.0, will clamp the + gradient to [-clamp, clamp]. + num_threads: Number of threads for OpenMP. + sigma: logit-undernormalization weight used in the multi-blank model. Refer to + the multi-blank paper https://arxiv.org/pdf/2211.03541 + for detailed explanations. """ minibatch_size = acts.shape[0] diff --git a/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py b/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py index c27a00f1b4e..7054f2abaf5 100644 --- a/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py +++ b/espnet2/asr/transducer/rnnt_multi_blank/rnnt_multi_blank.py @@ -37,21 +37,6 @@ class _RNNTNumba(Function): - """ - A custom autograd function for computing RNN Transducer (RNNT) loss using Numba. - - This class implements the forward and backward passes for the RNNT loss - computation, leveraging Numba for efficient CPU and GPU implementations. - It is designed to be used within PyTorch's autograd system. - - The class supports both standard RNNT loss and FastEmit regularization, - with options for different reduction methods and gradient clamping. - - Note: - This is an internal class and should not be instantiated directly. - Instead, use the `rnnt_loss` function or `RNNTLossNumba` module. - """ - @staticmethod def forward( ctx, @@ -64,34 +49,18 @@ def forward( fastemit_lambda, clamp, ): - """ - Forward pass for the RNN Transducer loss computation. - - This method computes the RNNT loss given the network outputs and labels. - It supports both CPU and GPU implementations. - - Args: - ctx (object): Context object to save information for backward pass. - acts (torch.Tensor): A 4D tensor (batch x seqLength x labelLength x outputDim) - containing output from network. - labels (torch.Tensor): 2D tensor containing all the targets of the batch - with zero padded. - act_lens (torch.Tensor): 1D tensor of size (batch) containing size of each - output sequence from the network. - label_lens (torch.Tensor): 1D tensor of (batch) containing label length - of each example. - blank (int): The blank label index. - reduction (str): Specifies the reduction to apply to the output. - fastemit_lambda (float): Scaling factor for FastEmit regularization. - clamp (float): Value for gradient clamping. - - Returns: - torch.Tensor: The computed RNNT loss. - - Note: - This method saves gradients in the context for use in the backward pass. - The actual loss computation is delegated to CUDA or CPU implementations - based on the input tensor's device. + """RNNTNumba Forward. + + log_probs: Tensor of (batch x seqLength x labelLength x outputDim) + containing output from network + labels: 2 dimensional Tensor containing all the targets of + the batch with zero padded + act_lens: Tensor of size (batch) containing size of each + output sequence from the network + label_lens: Tensor of (batch) containing label length of each example + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. """ is_cuda = acts.is_cuda @@ -132,52 +101,15 @@ def forward( @staticmethod def backward(ctx, grad_output): - """ - Backward pass for the RNN Transducer loss computation. - - This method computes the gradients of the RNNT loss with respect to the inputs. - - Args: - ctx (object): Context object containing saved tensors from the forward pass. - grad_output (torch.Tensor): Gradient of the loss with respect to the output - of the forward pass. - - Returns: - tuple: A tuple containing the gradients with respect to each input of the - forward function. The gradients for non-tensor inputs are None. - - Note: - This method relies on the gradients computed and saved during the forward pass. - It scales the saved gradients by the incoming gradient and returns them. - The gradient computation is automatically handled by PyTorch's autograd system. - """ if grad_output is not None and ctx.grads is not None: grad_output = grad_output.view(-1, 1, 1, 1).to(ctx.grads) return ctx.grads.mul_(grad_output), None, None, None, None, None, None, None class _MultiblankRNNTNumba(Function): - """ - A custom autograd function for computing Multi-blank RNN Transducer (RNNT) loss using Numba. + """Numba class for multi-blank transducer loss - This class implements the forward and backward passes for the Multi-blank RNNT loss - computation, leveraging Numba for efficient GPU implementations. It is designed to - be used within PyTorch's autograd system. - - The Multi-blank RNNT loss is an extension of the standard RNNT loss that incorporates - multiple blank symbols with different durations. This approach can improve the - performance of speech recognition systems, especially in streaming scenarios. - - The class supports both standard Multi-blank RNNT loss and FastEmit regularization, - with options for different reduction methods, gradient clamping, and logit - under-normalization. - - Note: - This is an internal class and should not be instantiated directly. - Instead, use the `multiblank_rnnt_loss` function or `MultiblankRNNTLossNumba` module. - - Reference: - https://arxiv.org/pdf/2211.03541.pdf + (https://arxiv.org/pdf/2211.03541.pdf) """ @staticmethod @@ -194,38 +126,15 @@ def forward( clamp, sigma, ): - """ - Forward pass for the Multi-blank RNN Transducer loss computation. - - This method computes the Multi-blank RNNT loss given the network outputs and labels. - It currently supports only GPU implementations. - - Args: - ctx (object): Context object to save information for backward pass. - acts (torch.Tensor): A 4D tensor (batch x seqLength x labelLength x outputDim) - containing output from network. - labels (torch.Tensor): 2D tensor containing all the targets of the batch - with zero padded. - act_lens (torch.Tensor): 1D tensor of size (batch) containing size of each - output sequence from the network. - label_lens (torch.Tensor): 1D tensor of (batch) containing label length - of each example. - blank (int): The standard blank label index. - big_blank_durations (list): List of durations for multi-blank transducer. - reduction (str): Specifies the reduction to apply to the output. - fastemit_lambda (float): Scaling factor for FastEmit regularization. - clamp (float): Value for gradient clamping. - sigma (float): Hyper-parameter for logit under-normalization method. - - Returns: - torch.Tensor: The computed Multi-blank RNNT loss. - - Raises: - NotImplementedError: If attempting to use CPU implementation. - - Note: - This method saves gradients in the context for use in the backward pass. - The actual loss computation is delegated to the GPU implementation. + """MultiblankRNNTNumba Forward. + + big_blank_durations: list of durations for multi-blank transducer, e.g. + [2, 4, 8]. + sigma: hyper-parameter for logit under-normalization method for training + multi-blank transducers. Recommended value 0.05. + Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for + the above parameters; + For other parameters for this class, refer to comment for class _RNNTNumba """ is_cuda = acts.is_cuda @@ -272,28 +181,6 @@ def forward( @staticmethod def backward(ctx, grad_output): - """ - Backward pass for the Multi-blank RNN Transducer loss computation. - - This method computes the gradients of the Multi-blank RNNT loss with respect to the inputs. - - Args: - ctx (object): Context object containing saved tensors from the forward pass. - grad_output (torch.Tensor): Gradient of the loss with respect to the output - of the forward pass. - - Returns: - tuple: A tuple containing the gradients with respect to each input of the - forward function. The gradients for non-tensor inputs are None. - - Note: - This method relies on the gradients computed and saved during the forward pass. - It scales the saved gradients by the incoming gradient and returns them. - The gradient computation is automatically handled by PyTorch's autograd system. - - The returned tuple has 11 elements to match the number of inputs in the forward method, - with None values for inputs that don't require gradients. - """ if grad_output is not None and ctx.grads is not None: grad_output = grad_output.view(-1, 1, 1, 1).to(ctx.grads) return ( @@ -321,45 +208,21 @@ def rnnt_loss( fastemit_lambda: float = 0.0, clamp: float = 0.0, ): - """ - Compute the RNN Transducer Loss. - - This function calculates the RNN Transducer Loss, which is commonly used in - speech recognition tasks. It supports both CPU and GPU computations. + """RNN Transducer Loss (functional form) Args: - acts (torch.Tensor): A 4D tensor of shape (batch, seqLength, labelLength, outputDim) - containing the output from the network. - labels (torch.Tensor): A 2D tensor containing all the targets of the batch - with zero padding. - act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of - each output sequence from the network. - label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label - length of each example. - blank (int, optional): The blank label index. Defaults to 0. - reduction (str, optional): Specifies the reduction to apply to the output. - Options are 'none', 'mean', or 'sum'. Defaults to 'mean'. - fastemit_lambda (float, optional): Scaling factor for FastEmit regularization. - Defaults to 0.0. - clamp (float, optional): Value for gradient clamping. If positive, gradients - will be clamped to [-clamp, clamp]. Defaults to 0.0. - - Returns: - torch.Tensor: The computed RNN Transducer Loss. - - Raises: - ValueError: If `clamp` is negative. - - Note: - For CPU computations, log_softmax is applied manually, while for GPU - computations, it's computed within the CUDA kernel. - - Example: - >>> acts = torch.randn(2, 10, 5, 20) - >>> labels = torch.randint(0, 19, (2, 5)) - >>> act_lens = torch.tensor([10, 8]) - >>> label_lens = torch.tensor([5, 4]) - >>> loss = rnnt_loss(acts, labels, act_lens, label_lens) + acts: Tensor of (batch x seqLength x labelLength x outputDim) + containing output from network + labels: 2 dimensional Tensor containing all the targets of + the batch with zero padded + act_lens: Tensor of size (batch) containing size of each + output sequence from the network + label_lens: Tensor of (batch) containing label length of each example + blank (int, optional): blank label. Default: 0. + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' """ if not acts.is_cuda: @@ -393,52 +256,28 @@ def multiblank_rnnt_loss( fastemit_lambda: float = 0.0, clamp: float = 0.0, ): - """ - Compute the Multi-blank RNN Transducer Loss. - - This function calculates the Multi-blank RNN Transducer Loss, which is an extension - of the standard RNN Transducer Loss that incorporates multiple blank symbols with - different durations. It is designed to improve the performance of speech recognition - systems, especially for streaming scenarios. + """Multi-blank RNN Transducer (https://arxiv.org/pdf/2211.03541.pdf) + Loss (functional form) Args: - acts (torch.Tensor): A 4D tensor of shape (batch, seqLength, labelLength, outputDim) - containing the output from the network. - labels (torch.Tensor): A 2D tensor containing all the targets of the batch - with zero padding. - act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of - each output sequence from the network. - label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label - length of each example. - blank (int): The standard blank label index. - big_blank_durations (list, optional): List of durations for multi-blank transducer, - e.g., [2, 4, 8]. Defaults to an empty list. - reduction (str, optional): Specifies the reduction to apply to the output. - Options are 'none', 'mean', or 'sum'. Defaults to 'mean'. - fastemit_lambda (float, optional): Scaling factor for FastEmit regularization. - Defaults to 0.0. - clamp (float, optional): Value for gradient clamping. If positive, gradients - will be clamped to [-clamp, clamp]. Defaults to 0.0. - - Returns: - torch.Tensor: The computed Multi-blank RNN Transducer Loss. - - Raises: - ValueError: If `clamp` is negative. - NotImplementedError: If trying to use CPU for computation (currently only GPU is supported). - - Note: - This implementation is based on the paper "Multi-blank Transducers for Speech Recognition" - (https://arxiv.org/pdf/2211.03541.pdf). It's designed to work with CUDA-enabled devices. - - Example: - >>> acts = torch.randn(2, 10, 5, 20).cuda() - >>> labels = torch.randint(0, 19, (2, 5)).cuda() - >>> act_lens = torch.tensor([10, 8]).cuda() - >>> label_lens = torch.tensor([5, 4]).cuda() - >>> blank = 0 - >>> big_blank_durations = [2, 4] - >>> loss = multiblank_rnnt_loss(acts, labels, act_lens, label_lens, blank, big_blank_durations) + acts: Tensor of (batch x seqLength x labelLength x outputDim) containing + output from network + labels: 2 dimensional Tensor containing all the targets of the batch with + zero padded + act_lens: Tensor of size (batch) containing size of each output + sequence from the network + label_lens: Tensor of (batch) containing label length of each example + blank (int): standard blank label. + big_blank_durations: list of durations for multi-blank transducer, e.g. + [2, 4, 8]. + sigma: hyper-parameter for logit under-normalization method for training + multi-blank transducers. Recommended value 0.05. + Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for + the last two params. + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' """ if not acts.is_cuda: @@ -470,28 +309,19 @@ def multiblank_rnnt_loss( class RNNTLossNumba(Module): - """ - A PyTorch module for computing RNN Transducer (RNNT) loss using Numba. - - This module provides an efficient implementation of the RNNT loss function, - leveraging Numba for improved performance. It supports both CPU and GPU - computations, with options for different reduction methods, FastEmit - regularization, and gradient clamping. - - The RNNT loss is commonly used in speech recognition tasks, particularly - for training end-to-end models. - - Attributes: - blank (int): The blank label index. Default is 0. - reduction (str): Specifies the reduction to apply to the output. - Can be 'none', 'mean', or 'sum'. Default is 'mean'. - fastemit_lambda (float): Scaling factor for FastEmit regularization. - Default is 0.0. - clamp (float): Value for gradient clamping. When set to a value >= 0.0, - will clamp the gradient to [-clamp, clamp]. Default is -1 (no clamping). - - Note: - This module uses the `_RNNTNumba` function for the actual loss computation. + """RNNT Loss Numba + + Parameters: + blank (int, optional): blank label. Default: 0. + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. + clamp: Float value. When set to value >= 0.0, will clamp the + gradient to [-clamp, clamp]. """ def __init__( @@ -505,30 +335,15 @@ def __init__( self.loss = _RNNTNumba.apply def forward(self, acts, labels, act_lens, label_lens): - """ - Forward pass for computing the RNN Transducer loss. - - This method calculates the RNNT loss given the network outputs and labels. - It handles both CPU and GPU implementations, applying necessary preprocessing - steps depending on the device. - - Args: - acts (torch.Tensor): A 4D tensor of shape (batch x seqLength x labelLength x outputDim) - containing output from the network. - labels (torch.Tensor): A 2D tensor containing all the targets of the batch - with zero padding. - act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of each - output sequence from the network. - label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label - length of each example. - - Returns: - torch.Tensor: The computed RNNT loss. - - Note: - For CPU computations, this method manually applies log_softmax and handles - gradient clamping if specified. For GPU computations, these operations are - performed within the CUDA kernel for efficiency. + """Forward RNNTLossNumba. + + log_probs: Tensor of (batch x seqLength x labelLength x outputDim) + containing output from network + labels: 2 dimensional Tensor containing all the targets of the + batch with zero padded + act_lens: Tensor of size (batch) containing size of each output + sequence from the network + label_lens: Tensor of (batch) containing label length of each example """ if not acts.is_cuda: @@ -559,32 +374,25 @@ def forward(self, acts, labels, act_lens, label_lens): class MultiblankRNNTLossNumba(Module): - """ - A PyTorch module for computing Multi-blank RNN Transducer (RNNT) loss using Numba. - - This module implements the Multi-blank RNNT loss function, which is an extension - of the standard RNNT loss that incorporates multiple blank symbols with different - durations. It is designed to improve the performance of speech recognition systems, - especially in streaming scenarios. - - The implementation uses Numba for efficient computation and currently supports - GPU operations only. - - Attributes: - blank (int): The standard blank label index. - big_blank_durations (list): List of durations for multi-blank transducer, e.g., [2, 4, 8]. - reduction (str): Specifies the reduction to apply to the output. - Can be 'none', 'mean', or 'sum'. Default is 'mean'. - fastemit_lambda (float): Scaling factor for FastEmit regularization. Default is 0.0. - clamp (float): Value for gradient clamping. When set to a value >= 0.0, - will clamp the gradient to [-clamp, clamp]. Default is -1 (no clamping). - sigma (float): Hyper-parameter for logit under-normalization method. Default is 0.0. - - Note: - This module uses the `_MultiblankRNNTNumba` function for the actual loss computation. - - Reference: - https://arxiv.org/pdf/2211.03541.pdf + """Multiblank RNNT Loss Numba + + Parameters: + blank (int): standard blank label. + big_blank_durations: list of durations for multi-blank transducer, e.g. + [2, 4, 8]. + sigma: hyper-parameter for logit under-normalization method for training + multi-blank transducers. Recommended value 0.05. + Refer to https://arxiv.org/pdf/2211.03541 for detailed explanations for + the above parameters; + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' + fastemit_lambda: Float scaling factor for FastEmit regularization. Refer to + FastEmit: Low-latency Streaming ASR with Sequence-level + Emission Regularization. + clamp: Float value. When set to value >= 0.0, will clamp the + gradient to [-clamp, clamp]. """ def __init__( @@ -606,33 +414,15 @@ def __init__( self.sigma = sigma def forward(self, acts, labels, act_lens, label_lens): - """ - Forward pass for computing the Multi-blank RNN Transducer loss. - - This method calculates the Multi-blank RNNT loss given the network outputs and labels. - It currently supports only GPU implementations, applying necessary preprocessing - steps before the loss computation. - - Args: - acts (torch.Tensor): A 4D tensor of shape (batch x seqLength x labelLength x outputDim) - containing output from the network. - labels (torch.Tensor): A 2D tensor containing all the targets of the batch - with zero padding. - act_lens (torch.Tensor): A 1D tensor of size (batch) containing the size of each - output sequence from the network. - label_lens (torch.Tensor): A 1D tensor of size (batch) containing the label - length of each example. - - Returns: - torch.Tensor: The computed Multi-blank RNNT loss. - - Raises: - NotImplementedError: If attempting to use CPU for computation. - - Note: - For GPU computations, this method manually applies log_softmax and handles - gradient clamping if specified. The actual loss computation is performed - within the CUDA kernel for efficiency. + """MultiblankRNNTLossNumba Forward. + + log_probs: Tensor of (batch x seqLength x labelLength x outputDim) + containing output from network + labels: 2 dimensional Tensor containing all the targets of + the batch with zero padded + act_lens: Tensor of size (batch) containing size of each output + sequence from the network + label_lens: Tensor of (batch) containing label length of each example """ if not acts.is_cuda: @@ -665,115 +455,23 @@ def forward(self, acts, labels, act_lens, label_lens): def check_type(var, t, name): - """ - Check if a variable has the expected data type. - - This function verifies whether the given variable has the specified data type. - If the variable's type doesn't match the expected type, it raises a TypeError. - - Args: - var (Any): The variable to check. - t (type): The expected data type. - name (str): The name of the variable (used in the error message). - - Raises: - TypeError: If the variable's type doesn't match the expected type. - - Example: - >>> import torch - >>> tensor = torch.tensor([1, 2, 3]) - >>> check_type(tensor, torch.Tensor, "tensor") - >>> check_type(tensor, torch.float32, "tensor") - TypeError: tensor must be torch.float32 - """ if var.dtype is not t: raise TypeError("{} must be {}".format(name, t)) def check_contiguous(var, name): - """ - Check if a tensor is contiguous in memory. - - This function verifies whether the given tensor is contiguous in memory. - If the tensor is not contiguous, it raises a ValueError. - - Args: - var (torch.Tensor): The tensor to check for contiguity. - name (str): The name of the tensor (used in the error message). - - Raises: - ValueError: If the tensor is not contiguous in memory. - - Example: - >>> import torch - >>> tensor = torch.tensor([[1, 2], [3, 4]]) - >>> check_contiguous(tensor, "tensor") - >>> non_contiguous = tensor.t() - >>> check_contiguous(non_contiguous, "non_contiguous") - ValueError: non_contiguous must be contiguous - """ if not var.is_contiguous(): raise ValueError("{} must be contiguous".format(name)) def check_dim(var, dim, name): - """ - Check if a tensor has the expected number of dimensions. - - This function verifies whether the given tensor has the specified number of dimensions. - If the tensor's dimensionality doesn't match the expected value, it raises a ValueError. - - Args: - var (torch.Tensor): The tensor to check. - dim (int): The expected number of dimensions. - name (str): The name of the tensor (used in the error message). - - Raises: - ValueError: If the tensor's number of dimensions doesn't match the expected value. - - Example: - >>> import torch - >>> tensor_2d = torch.tensor([[1, 2], [3, 4]]) - >>> check_dim(tensor_2d, 2, "tensor_2d") - >>> tensor_3d = torch.ones(2, 3, 4) - >>> check_dim(tensor_3d, 2, "tensor_3d") - ValueError: tensor_3d must be 2D - """ if len(var.shape) != dim: raise ValueError("{} must be {}D".format(name, dim)) def certify_inputs(log_probs, labels, lengths, label_lengths): # check_type(log_probs, torch.float32, "log_probs") - """ - Certify that the input tensors meet the required specifications for RNNT loss computation. - - This function performs a series of checks on the input tensors to ensure they meet - the necessary requirements for computing the RNNT loss. It verifies data types, - contiguity, dimensions, and shape consistency. - - Args: - log_probs (torch.Tensor): A 4D tensor of log probabilities from the network output. - labels (torch.Tensor): A 2D tensor of label sequences. - lengths (torch.Tensor): A 1D tensor of input sequence lengths. - label_lengths (torch.Tensor): A 1D tensor of label sequence lengths. - - Raises: - TypeError: If any tensor has an incorrect data type. - ValueError: If any tensor is not contiguous, has incorrect dimensions, - or if there's a mismatch in batch sizes or sequence lengths. - - Note: - This function is typically called internally by the RNNT loss function - to validate inputs before computation. - - Example: - >>> log_probs = torch.randn(2, 10, 5, 20, dtype=torch.float32) - >>> labels = torch.randint(0, 19, (2, 5), dtype=torch.int32) - >>> lengths = torch.tensor([10, 8], dtype=torch.int32) - >>> label_lengths = torch.tensor([5, 4], dtype=torch.int32) - >>> certify_inputs(log_probs, labels, lengths, label_lengths) - """ + check_type(labels, torch.int32, "labels") check_type(label_lengths, torch.int32, "label_lengths") check_type(lengths, torch.int32, "lengths") check_contiguous(log_probs, "log_probs") From 1afb9c8f295f921e914eb79c97d86ffab3a280c2 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Mon, 26 Aug 2024 14:41:59 -0400 Subject: [PATCH 35/57] Fixed docstring in the espnetez - so that all lines fit within 88 characters to avoid CI error --- espnetez/config.py | 26 +++++++++++++++----------- espnetez/dataset.py | 28 ++++++++++++++++------------ espnetez/task.py | 6 ++++-- espnetez/trainer.py | 15 ++++++++++----- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/espnetez/config.py b/espnetez/config.py index d923f9ee520..d1d144f45e9 100644 --- a/espnetez/config.py +++ b/espnetez/config.py @@ -5,7 +5,7 @@ def convert_none_to_None(dic): """ - Recursively convert string representations of 'none' in a dictionary to Python's None. + Recursively convert string representations of 'none' in a dictionary to None. This function traverses a dictionary and replaces any occurrences of the string "none" with the actual Python None type. If a value in the dictionary is another @@ -20,7 +20,8 @@ def convert_none_to_None(dic): by None. Examples: - >>> sample_dict = {'key1': 'none', 'key2': {'subkey1': 'none', 'subkey2': 'value'}} + >>> sample_dict = {'key1': 'none', 'key2': {'subkey1': 'none', + 'subkey2': 'value'}} >>> convert_none_to_None(sample_dict) {'key1': None, 'key2': {'subkey1': None, 'subkey2': 'value'}} @@ -43,11 +44,11 @@ def convert_none_to_None(dic): def from_yaml(task, path): """ - Load configuration from a YAML file and merge it with the default task configuration. + Load configuration from a YAML file and merge it with the default configuration. This function reads a YAML configuration file from the specified path and merges its contents with the default configuration for the specified task. If there are any - keys in the YAML file that have the string value "none", they are converted to Python's + keys in the YAML file that have the string value "none", they are converted to `None` type. The resulting configuration dictionary is returned. Args: @@ -99,12 +100,14 @@ def update_finetune_config(task, pretrain_config, path): Args: task (str): The name of the task for which the configuration is being updated. - pretrain_config (dict): The existing pre-training configuration dictionary to be updated. - path (str): The file path to the YAML file containing the fine-tuning configuration. + pretrain_config (dict): The existing pre-training configuration dictionary + to be updated. + path (str): The file path to the YAML file containing the fine-tuning + configuration. Returns: - dict: The updated pre-training configuration dictionary after merging with the fine-tuning - configuration and defaults from the specified task. + dict: The updated pre-training configuration dictionary after merging with the + fine-tuning configuration and defaults from the specified task. Examples: >>> pretrain_cfg = { @@ -112,7 +115,8 @@ def update_finetune_config(task, pretrain_config, path): ... "batch_size": 32, ... "dist_backend": "nccl" ... } - >>> updated_cfg = update_finetune_config("asr", pretrain_cfg, "finetune_config.yaml") + >>> updated_cfg = update_finetune_config("asr", pretrain_cfg, + "finetune_config.yaml") >>> print(updated_cfg) { "learning_rate": 0.0001, # updated from finetune_config.yaml @@ -126,8 +130,8 @@ def update_finetune_config(task, pretrain_config, path): yaml.YAMLError: If the YAML file is improperly formatted. Note: - The function assumes that the task class provides a method `get_default_config()` - which returns the default configuration as a dictionary. + The function assumes that the task class provides a method + `get_default_config()` which returns the default configuration as a dictionary. """ with open(path, "r") as f: finetune_config = yaml.load(f, Loader=yaml.Loader) diff --git a/espnetez/dataset.py b/espnetez/dataset.py index 4a1a9c73450..0bb92cb2f0e 100644 --- a/espnetez/dataset.py +++ b/espnetez/dataset.py @@ -30,7 +30,10 @@ class ESPnetEZDataset(AbsDataset): __len__() -> int: Returns the total number of entries in the dataset. Examples: - >>> dataset = [("audio1.wav", "transcription1"), ("audio2.wav", "transcription2")] + >>> dataset = [ + ("audio1.wav", "transcription1"), + ("audio2.wav", "transcription2") + ] >>> data_info = { ... "audio": lambda x: x[0], ... "transcription": lambda x: x[1] @@ -69,7 +72,8 @@ def has_name(self, name) -> bool: bool: True if the name exists in the data information; False otherwise. Examples: - >>> dataset = ESPnetEZDataset(dataset=[...], data_info={'feature1': ..., 'feature2': ...}) + >>> dataset = ESPnetEZDataset(dataset=[...], + data_info={'feature1': ..., 'feature2': ...}) >>> dataset.has_name('feature1') True >>> dataset.has_name('feature3') @@ -90,15 +94,15 @@ def names(self) -> Tuple[str, ...]: metadata, allowing for efficient data access and manipulation. Attributes: - dataset (Union[list, tuple]): The underlying dataset that contains the data entries. - data_info (Dict[str, callable]): A dictionary mapping names to functions that process - each data entry in the dataset. + dataset (Union[list, tuple]): The underlying dataset that contains the data. + data_info (Dict[str, callable]): A dictionary mapping names to functions + that process each data entry in the dataset. Args: dataset (Union[list, tuple]): The dataset to be wrapped. - data_info (Dict[str, callable]): A dictionary where keys are the names of the data - attributes and values are functions that extract or transform the data from the - dataset. + data_info (Dict[str, callable]): A dictionary where keys are the names of + the data attributes and values are functions that extract or transform + the data from the dataset. Methods: has_name(name: str) -> bool: @@ -114,8 +118,8 @@ def names(self) -> Tuple[str, ...]: Returns the number of entries in the dataset. Examples: - >>> dataset = ESPnetEZDataset(dataset=[...], data_info={'feature': lambda x: x.feature, - ... 'label': lambda x: x.label}) + >>> dataset = ESPnetEZDataset(dataset=[...], + data_info={'feature': lambda x: x.feature, 'label': lambda x: x.label}) >>> dataset.has_name('feature') True >>> dataset.names() @@ -127,8 +131,8 @@ def names(self) -> Tuple[str, ...]: 100 Note: - The functions provided in the data_info should be callable and should accept a single - argument corresponding to an entry from the dataset. + The functions provided in the data_info should be callable and should + accept a single argument corresponding to an entry from the dataset. """ return tuple(self.data_info.keys()) diff --git a/espnetez/task.py b/espnetez/task.py index 8d8638d07b6..74ca54d39b4 100644 --- a/espnetez/task.py +++ b/espnetez/task.py @@ -138,7 +138,8 @@ def get_ez_task_with_dataset(task_name: str) -> AbsTask: in the ESPnet framework, such as 'asr', 'tts', etc. Returns: - AbsTask: A subclass of AbsTask that supports custom datasets for the specified task. + AbsTask: A subclass of AbsTask that supports custom datasets for the + specified task. Examples: >>> from espnetez.task import get_ez_task_with_dataset @@ -146,7 +147,8 @@ def get_ez_task_with_dataset(task_name: str) -> AbsTask: >>> custom_asr_task.train_dataset = my_custom_train_dataset >>> custom_asr_task.valid_dataset = my_custom_valid_dataset >>> model = custom_asr_task.build_model(args) - >>> iterator = custom_asr_task.build_iter_factory(args, distributed_option, mode='train') + >>> iterator = custom_asr_task.build_iter_factory(args, distributed_option, + mode='train') Note: Ensure that the specified task name is valid and that the corresponding diff --git a/espnetez/trainer.py b/espnetez/trainer.py index 09f885d0c03..2f74a8855b3 100644 --- a/espnetez/trainer.py +++ b/espnetez/trainer.py @@ -44,14 +44,16 @@ def check_argument( Examples: # Example of valid usage with dump files - check_argument('/path/to/train_dump', '/path/to/valid_dump', None, None, None, None) + check_argument('/path/to/train_dump', '/path/to/valid_dump', None, None, None, + None) # Example of valid usage with datasets check_argument(None, None, train_dataset, valid_dataset, None, None) # Example of invalid usage - mix of dump files and datasets check_argument('/path/to/train_dump', None, train_dataset, None, None, None) - # Raises ValueError: If you try to use dump file, dataset or dataloader should be None. + # Raises ValueError: If you try to use dump file, dataset or dataloader should + be None. Note: Ensure to specify at least one of the arguments: dump directories, @@ -124,9 +126,11 @@ class Trainer: consistent and valid before starting the training process. Attributes: - train_config (Namespace): Configuration for training, can be a dictionary or Namespace object. + train_config (Namespace): Configuration for training, can be a dictionary or + Namespace object. task_class (Task): Task class instantiated from the provided task identifier. - stats_dir (str): Directory where statistics for training and validation will be stored. + stats_dir (str): Directory where statistics for training and validation will be + stored. output_dir (str): Directory where model outputs will be saved. Args: @@ -291,7 +295,8 @@ def collect_stats(self): Examples: >>> trainer = Trainer(task='example_task', train_config=some_config, - ... output_dir='/path/to/output', stats_dir='/path/to/stats') + ... output_dir='/path/to/output', + ... stats_dir='/path/to/stats') >>> trainer.collect_stats() Note: From 211c3bd7880d0ac3ddea0069f6e820846d5e7c62 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:43:12 +0000 Subject: [PATCH 36/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- espnetez/config.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/espnetez/config.py b/espnetez/config.py index d1d144f45e9..e2859c4c738 100644 --- a/espnetez/config.py +++ b/espnetez/config.py @@ -20,7 +20,7 @@ def convert_none_to_None(dic): by None. Examples: - >>> sample_dict = {'key1': 'none', 'key2': {'subkey1': 'none', + >>> sample_dict = {'key1': 'none', 'key2': {'subkey1': 'none', 'subkey2': 'value'}} >>> convert_none_to_None(sample_dict) {'key1': None, 'key2': {'subkey1': None, 'subkey2': 'value'}} @@ -48,7 +48,7 @@ def from_yaml(task, path): This function reads a YAML configuration file from the specified path and merges its contents with the default configuration for the specified task. If there are any - keys in the YAML file that have the string value "none", they are converted to + keys in the YAML file that have the string value "none", they are converted to `None` type. The resulting configuration dictionary is returned. Args: @@ -100,13 +100,13 @@ def update_finetune_config(task, pretrain_config, path): Args: task (str): The name of the task for which the configuration is being updated. - pretrain_config (dict): The existing pre-training configuration dictionary + pretrain_config (dict): The existing pre-training configuration dictionary to be updated. - path (str): The file path to the YAML file containing the fine-tuning + path (str): The file path to the YAML file containing the fine-tuning configuration. Returns: - dict: The updated pre-training configuration dictionary after merging with the + dict: The updated pre-training configuration dictionary after merging with the fine-tuning configuration and defaults from the specified task. Examples: @@ -130,7 +130,7 @@ def update_finetune_config(task, pretrain_config, path): yaml.YAMLError: If the YAML file is improperly formatted. Note: - The function assumes that the task class provides a method + The function assumes that the task class provides a method `get_default_config()` which returns the default configuration as a dictionary. """ with open(path, "r") as f: From 345cc309fdd054c042e6af843db530d6cd58965a Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 15:48:36 -0400 Subject: [PATCH 37/57] Make ci happy --- ci/doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/doc.sh b/ci/doc.sh index 30e795edccb..f8324b6a107 100755 --- a/ci/doc.sh +++ b/ci/doc.sh @@ -24,8 +24,8 @@ build_and_convert () { # $1: path # $2: output mkdir -p ./doc/_gen/tools/$2 - for filename in `find $1`; do - bn=`basename ${filename}` + for filename in $1; do + bn=$(basename ${filename}) echo "Converting ${filename} to rst..." ./doc/usage2rst.sh ${filename} > ./doc/_gen/tools/$2/${bn}.rst done From dccb20ca0c7dfc4e52f1c165ba21e599878b4f22 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 16:26:17 -0400 Subject: [PATCH 38/57] Revert original dependency --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index d50833a7831..1041d49db15 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,10 @@ "asteroid_filterbanks==0.4.0", # UASR "editdistance", + # fix CI error due to the use of deprecated functions + # https://github.com/espnet/espnet/actions/runs/3174416926/jobs/5171182884#step:8:8419 + # https://importlib-metadata.readthedocs.io/en/latest/history.html#v5-0-0 + "importlib-metadata<5.0", ], # train: The modules invoked when training only. "train": [ From 024691964b30cff6af174fd0c398b6412a221419 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 16:27:18 -0400 Subject: [PATCH 39/57] Revert original dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1041d49db15..4a2cf64cd60 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,6 @@ "sphinx-markdown-tables>=0.0.12", "jupyter", "sphinx-markdown-builder", - "importlib-metadata", ], } requirements["all"].extend(requirements["train"] + requirements["recipe"]) From 12e5e244b516a5179969fdde0e07619890cec91e Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 16:40:09 -0400 Subject: [PATCH 40/57] numpy recommends <60 setuptools --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a2cf64cd60..1ba6565e73e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ requirements = { "install": [ - "setuptools>=38.5.1", + "setuptools>=38.5.1,<60.0", "packaging", "configargparse>=1.2.1", "typeguard", From f3cba8842fe551d76b97353ebcb47e871c74adac Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 18:12:25 -0400 Subject: [PATCH 41/57] gtn has newer ver available --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1ba6565e73e..3fa79bfb8e9 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ "torch_optimizer", "fairscale", "transformers", - "gtn==0.0.0", + "gtn==0.0.1", "evaluate", ], "setup": [ From 53657da1fbabbd81a4e60db4159eb2ad35f43706 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 18:28:53 -0400 Subject: [PATCH 42/57] gtn 0.0.1 fails to install --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3fa79bfb8e9..1ba6565e73e 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ "torch_optimizer", "fairscale", "transformers", - "gtn==0.0.1", + "gtn==0.0.0", "evaluate", ], "setup": [ From 1003c9418e9884813547f58af3a951a02a4da618 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 18:52:45 -0400 Subject: [PATCH 43/57] upgrade setuptools --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1ba6565e73e..4a2cf64cd60 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ requirements = { "install": [ - "setuptools>=38.5.1,<60.0", + "setuptools>=38.5.1", "packaging", "configargparse>=1.2.1", "typeguard", From 50b3b6e2d80fe83f6c684cca3bfa1d42b336e6ec Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Tue, 27 Aug 2024 19:10:08 -0400 Subject: [PATCH 44/57] revert mistake --- espnet2/asr/decoder/transformer_decoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/espnet2/asr/decoder/transformer_decoder.py b/espnet2/asr/decoder/transformer_decoder.py index 7219b801b36..02b1febbc09 100644 --- a/espnet2/asr/decoder/transformer_decoder.py +++ b/espnet2/asr/decoder/transformer_decoder.py @@ -28,7 +28,9 @@ ) -class BaseTransformerDecoder(AbsDecoder, BatchScorerInterface): +class BaseTransformerDecoder( + AbsDecoder, BatchScorerInterface, MaskParallelScorerInterface +): """Base class of Transfomer decoder module. Args: From 1c31c016c3529f6c96ae455b2c6b80015bf65b6e Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 27 Aug 2024 19:30:11 -0400 Subject: [PATCH 45/57] Changed jupyter to jupyterlab --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a2cf64cd60..782f68d9cbb 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", - "jupyter", + "jupyterlab=4.2.4", "sphinx-markdown-builder", ], } From fafdc4f3399e7a342e86ff43d284faf30a244d1c Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 27 Aug 2024 19:37:32 -0400 Subject: [PATCH 46/57] Fixed typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 782f68d9cbb..084b926f834 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", - "jupyterlab=4.2.4", + "jupyterlab==4.2.4", "sphinx-markdown-builder", ], } From d632e4eb144c80200bf165a65da8487712791d23 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 27 Aug 2024 20:08:33 -0400 Subject: [PATCH 47/57] Fixed dependency version - IT seems that the setuptools removed something related to `distutils._msvccompiler` today. This might cause the error. https://setuptools.pypa.io/en/stable/history.html#deprecations-and-removals --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 084b926f834..04d7052da65 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ requirements = { "install": [ - "setuptools>=38.5.1", + "setuptools==73.0.1", "packaging", "configargparse>=1.2.1", "typeguard", @@ -114,7 +114,7 @@ "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", - "jupyterlab==4.2.4", + "jupyterlab==4", "sphinx-markdown-builder", ], } From 43a3b145d38d7e4e7eb0c5de653ce223734d73d3 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 27 Aug 2024 20:11:49 -0400 Subject: [PATCH 48/57] Fix mistake --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 04d7052da65..d7caa0a2a7c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ requirements = { "install": [ - "setuptools==73.0.1", + "setuptools>=38.5.1,<74.0.0", "packaging", "configargparse>=1.2.1", "typeguard", From f119a02f26437fb9eab5851e99006304fe76fe43 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Tue, 27 Aug 2024 23:18:34 -0400 Subject: [PATCH 49/57] Bug fix for document generation - The `...` characters in the docstring cause this error, if the `...` is included inside the dictionary. --- espnet2/asr/state_spaces/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/espnet2/asr/state_spaces/utils.py b/espnet2/asr/state_spaces/utils.py index 7234cf07888..029816e5302 100644 --- a/espnet2/asr/state_spaces/utils.py +++ b/espnet2/asr/state_spaces/utils.py @@ -19,7 +19,7 @@ def is_dict(x): def to_dict(x, recursive=True): """Convert Sequence or Mapping object to dict. - lists get converted to {0: x[0], 1: x[1], ...} + lists get converted to {0: x[0], 1: x[1]} """ if is_list(x): x = {i: v for i, v in enumerate(x)} From 386b291df6e4db4272bddddb19a4ca4877186950 Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 28 Aug 2024 00:01:35 -0400 Subject: [PATCH 50/57] fix bug in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7caa0a2a7c..6e4857094f6 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ "myst-parser", "nbsphinx>=0.4.2", "sphinx-markdown-tables>=0.0.12", - "jupyterlab==4", + "jupyterlab<5", "sphinx-markdown-builder", ], } From adfbfe4d1909a5ec1d4ed88c61b1be68169bd18d Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 28 Aug 2024 11:36:34 -0400 Subject: [PATCH 51/57] change doc installation procedure --- doc/README.md | 1 + setup.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/README.md b/doc/README.md index 38278f9530e..2b2cf5b397b 100644 --- a/doc/README.md +++ b/doc/README.md @@ -11,6 +11,7 @@ $ conda create -p ./envs python=3.10 $ conda activate ./envs # Requirements +$ pip install -e ".[all]" $ pip install -e ".[doc]" $ conda install conda-forge::ffmpeg $ conda install conda-forge::nodejs==22.6.0 diff --git a/setup.py b/setup.py index 6e4857094f6..e5e2405c4d2 100644 --- a/setup.py +++ b/setup.py @@ -120,7 +120,6 @@ } requirements["all"].extend(requirements["train"] + requirements["recipe"]) requirements["test"].extend(requirements["train"]) -requirements["doc"].extend(requirements["all"]) install_requires = requirements["install"] setup_requires = requirements["setup"] From 868cad50a35d8af5837a62c18318e3500defd33a Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 28 Aug 2024 12:40:22 -0400 Subject: [PATCH 52/57] Removed debug code and default images from VuePress-Hope. - Removed debug code. - Removed several images that is not used in our homepage. --- doc/usage2rst.sh | 3 +-- doc/vuepress/src/.vuepress/config.ts | 2 +- doc/vuepress/src/.vuepress/public/assets/image/advanced.svg | 1 - doc/vuepress/src/.vuepress/public/assets/image/blog.svg | 1 - doc/vuepress/src/.vuepress/public/assets/image/features.svg | 1 - doc/vuepress/src/.vuepress/public/assets/image/layout.svg | 1 - doc/vuepress/src/.vuepress/public/assets/image/markdown.svg | 1 - 7 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 doc/vuepress/src/.vuepress/public/assets/image/advanced.svg delete mode 100644 doc/vuepress/src/.vuepress/public/assets/image/blog.svg delete mode 100644 doc/vuepress/src/.vuepress/public/assets/image/features.svg delete mode 100644 doc/vuepress/src/.vuepress/public/assets/image/layout.svg delete mode 100644 doc/vuepress/src/.vuepress/public/assets/image/markdown.svg diff --git a/doc/usage2rst.sh b/doc/usage2rst.sh index 2cfa7a30098..da6719e9a4a 100755 --- a/doc/usage2rst.sh +++ b/doc/usage2rst.sh @@ -8,8 +8,7 @@ if [ $1 == "--help" ]; then fi real=$(realpath $1) -# githash=`git rev-parse HEAD` -githash=8574aa8e0d5968916df1df42c28621a48b2a5281 +githash=`git rev-parse HEAD` cd ./egs2/wsj/asr1 . path.sh diff --git a/doc/vuepress/src/.vuepress/config.ts b/doc/vuepress/src/.vuepress/config.ts index d1cce04bd0a..b0f0b195eee 100644 --- a/doc/vuepress/src/.vuepress/config.ts +++ b/doc/vuepress/src/.vuepress/config.ts @@ -4,7 +4,7 @@ import { viteBundler } from "@vuepress/bundler-vite"; import theme from "./theme.js"; export default defineUserConfig({ - base: "/espnet_draft_home_page/", + base: "/espnet/", lang: "en-US", description: "A documentation for ESPnet", diff --git a/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg b/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg deleted file mode 100644 index ba4156d0150..00000000000 --- a/doc/vuepress/src/.vuepress/public/assets/image/advanced.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/vuepress/src/.vuepress/public/assets/image/blog.svg b/doc/vuepress/src/.vuepress/public/assets/image/blog.svg deleted file mode 100644 index 896ddb3c811..00000000000 --- a/doc/vuepress/src/.vuepress/public/assets/image/blog.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/vuepress/src/.vuepress/public/assets/image/features.svg b/doc/vuepress/src/.vuepress/public/assets/image/features.svg deleted file mode 100644 index ceaded1de5d..00000000000 --- a/doc/vuepress/src/.vuepress/public/assets/image/features.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/vuepress/src/.vuepress/public/assets/image/layout.svg b/doc/vuepress/src/.vuepress/public/assets/image/layout.svg deleted file mode 100644 index d60be4cd53d..00000000000 --- a/doc/vuepress/src/.vuepress/public/assets/image/layout.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg b/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg deleted file mode 100644 index 28973388eb6..00000000000 --- a/doc/vuepress/src/.vuepress/public/assets/image/markdown.svg +++ /dev/null @@ -1 +0,0 @@ - From dca09ae3c1f05c45984bbd048bc26b96764851a5 Mon Sep 17 00:00:00 2001 From: Masao-Someki Date: Wed, 28 Aug 2024 13:10:27 -0400 Subject: [PATCH 53/57] Fix CI bug --- doc/usage2rst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage2rst.sh b/doc/usage2rst.sh index da6719e9a4a..c8d99c319e6 100755 --- a/doc/usage2rst.sh +++ b/doc/usage2rst.sh @@ -8,7 +8,7 @@ if [ $1 == "--help" ]; then fi real=$(realpath $1) -githash=`git rev-parse HEAD` +githash=$(git rev-parse HEAD) cd ./egs2/wsj/asr1 . path.sh From dbaa63aea5d751405a55e9f9060bcf2efbfca6fb Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 28 Aug 2024 13:35:23 -0400 Subject: [PATCH 54/57] Update about_this_doc.md --- doc/about_this_doc.md | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/doc/about_this_doc.md b/doc/about_this_doc.md index b33577e14ad..99052dc4f8a 100644 --- a/doc/about_this_doc.md +++ b/doc/about_this_doc.md @@ -64,24 +64,3 @@ This document outlines the process of automatically generating the ESPnet homepa 10. **Finalize the Build**: Install Node.js and the necessary dependencies, then build the homepage. To preview the page, comment out the `docs:build` line and uncomment the `docs:dev` line in the script. - -11. **Create a `clean.sh` Script for Development**: - If you need to clear the build cache during development, you can use the following script: - ```bash - #!/bin/bash - - rm -r dist - rm -r espnet_bin - rm -r espnet2_bin - rm -r utils_py - - rm -r doc/_gen - rm -r doc/build - - rm -r doc/vuepress/src/*.md - find doc/vuepress/src/notebook -type f -exec chmod 777 {} \; - rm -r doc/vuepress/src/notebook - rm -r doc/vuepress/src/* - rm -r doc/vuepress/src/.vuepress/.temp - rm -r doc/vuepress/src/.vuepress/.cache - ``` From e7e0c918ee4c1737e4e4de6f6e325317264a618a Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 28 Aug 2024 13:37:12 -0400 Subject: [PATCH 55/57] Refactor argparse2rst.py --- doc/argparse2rst.py | 70 ++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/doc/argparse2rst.py b/doc/argparse2rst.py index 3fef85fb1c7..f94dccfdb6e 100755 --- a/doc/argparse2rst.py +++ b/doc/argparse2rst.py @@ -52,47 +52,45 @@ def get_parser(): return parser -# parser -args = get_parser().parse_args() - - -modinfo = [] - -for p in args.src: - if "__init__.py" in p: - continue - try: - modinfo.append(ModuleInfo(p)) - except Exception as e: - logging.error(f"Error processing {p}: {str(e)}") - -print(f""" +if __name__ == "__main__": + # parser + args = get_parser().parse_args() + + modinfo = [] + for p in args.src: + if "__init__.py" in p: + continue + try: + modinfo.append(ModuleInfo(p)) + except Exception as e: + logging.error(f"Error processing {p}: {str(e)}") + + print(f""" {args.title} {"=" * len(args.title)} """) -for m in modinfo: - logging.info(f"processing: {m.path.name}") - d = m.module.get_parser().description - assert d is not None - print(f"- :ref:`{m.path.name}`: {d}") - -print() - -os.makedirs(args.output_dir, exist_ok=True) - -# print argparse to each files -for m in modinfo: - cmd = m.path.name - sourceurl = f"https://github.com/espnet/espnet/blob/" \ - + get_git_revision_hash() + str(m.path.parent / m.path.stem) + ".py" - sep = "~" * len(cmd) - mname = m.name if m.name.startswith("espnet") \ - else ".".join(m.name.split(".")[1:]) - with open(f"{args.output_dir}/{cmd[:-3]}.rst", "w") as writer: # remove .py - writer.write( - f""".. _{cmd} + for m in modinfo: + logging.info(f"processing: {m.path.name}") + d = m.module.get_parser().description + assert d is not None + print(f"- :ref:`{m.path.name}`: {d}") + + print() + + os.makedirs(args.output_dir, exist_ok=True) + + # print argparse to each files + for m in modinfo: + cmd = m.path.name + sourceurl = f"https://github.com/espnet/espnet/blob/" \ + + get_git_revision_hash() + str(m.path.parent / m.path.stem) + ".py" + sep = "~" * len(cmd) + mname = m.name if m.name.startswith("espnet") \ + else ".".join(m.name.split(".")[1:]) + with open(f"{args.output_dir}/{cmd[:-3]}.rst", "w") as writer: # remove .py + writer.write(f""".. _{cmd} {cmd} {sep} From 80dd26705d1266c4c808c48950a85a58ead30c20 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:39:55 +0000 Subject: [PATCH 56/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/argparse2rst.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/argparse2rst.py b/doc/argparse2rst.py index f94dccfdb6e..310b9c66cd2 100755 --- a/doc/argparse2rst.py +++ b/doc/argparse2rst.py @@ -76,11 +76,11 @@ def get_parser(): d = m.module.get_parser().description assert d is not None print(f"- :ref:`{m.path.name}`: {d}") - + print() - + os.makedirs(args.output_dir, exist_ok=True) - + # print argparse to each files for m in modinfo: cmd = m.path.name From 026400393299eb5624e63523137cb93b72dab5eb Mon Sep 17 00:00:00 2001 From: Kwanghee Choi Date: Wed, 28 Aug 2024 13:43:49 -0400 Subject: [PATCH 57/57] Refactor minor stuff --- doc/convert_custom_tags_to_html.py | 63 ++++++++++---------- doc/convert_md_to_homepage.py | 15 ++--- doc/members2rst.py | 96 +++++++++++++++--------------- 3 files changed, 89 insertions(+), 85 deletions(-) diff --git a/doc/convert_custom_tags_to_html.py b/doc/convert_custom_tags_to_html.py index 9dee9e38819..7d0496e6746 100644 --- a/doc/convert_custom_tags_to_html.py +++ b/doc/convert_custom_tags_to_html.py @@ -211,34 +211,35 @@ def replace_language_tags(content): return content -# parser -args = get_parser().parse_args() - -for md in glob.glob(f"{args.root}/*.md", recursive=True): - with open(md, "r") as f: - content = f.read() - - # Replace the "" and "" with "<" and ">", respectively - # if the tag is not in ALL_HTML_TAGS and does not have its end tag - # we need to apply this two functions because - # there are custom tags like: "" - content = replace_language_tags(content) - content = replace_string_tags(content) - content = replace_custom_tags(content) - - with open(md, "w") as f: - f.write(content) - - -for md in glob.glob(f"{args.root}/**/*.md", recursive=True): - with open(md, "r") as f: - content = f.read() - - # Replace the "" and "" with "<" and ">", respectively - # if the tag is not in ALL_HTML_TAGS - content = replace_language_tags(content) - content = replace_string_tags(content) - content = replace_custom_tags(content) - - with open(md, "w") as f: - f.write(content) +if __name__ == "__main__": + # parser + args = get_parser().parse_args() + + for md in glob.glob(f"{args.root}/*.md", recursive=True): + with open(md, "r") as f: + content = f.read() + + # Replace the "" and "" with "<" and ">", respectively + # if the tag is not in ALL_HTML_TAGS and does not have its end tag + # we need to apply this two functions because + # there are custom tags like: "" + content = replace_language_tags(content) + content = replace_string_tags(content) + content = replace_custom_tags(content) + + with open(md, "w") as f: + f.write(content) + + + for md in glob.glob(f"{args.root}/**/*.md", recursive=True): + with open(md, "r") as f: + content = f.read() + + # Replace the "" and "" with "<" and ">", respectively + # if the tag is not in ALL_HTML_TAGS + content = replace_language_tags(content) + content = replace_string_tags(content) + content = replace_custom_tags(content) + + with open(md, "w") as f: + f.write(content) diff --git a/doc/convert_md_to_homepage.py b/doc/convert_md_to_homepage.py index 79ed1172c5f..a93abfffd71 100644 --- a/doc/convert_md_to_homepage.py +++ b/doc/convert_md_to_homepage.py @@ -76,11 +76,12 @@ def convert(markdown_text): return _result -# parser -args = get_parser().parse_args() +if __name__ == "__main__": + # parser + args = get_parser().parse_args() -for md in glob(f"{args.root}/**/*.md", recursive=True): - markdown_text = open(md, "r").read() - _result = convert(markdown_text) - with open(md, "w") as f: - f.write(_result) + for md in glob(f"{args.root}/**/*.md", recursive=True): + markdown_text = open(md, "r").read() + _result = convert(markdown_text) + with open(md, "w") as f: + f.write(_result) diff --git a/doc/members2rst.py b/doc/members2rst.py index 68a2586fc78..e094a92f203 100644 --- a/doc/members2rst.py +++ b/doc/members2rst.py @@ -67,50 +67,52 @@ def gen_class_rst(class_name, writer, filepath, lineno): :show-inheritance: """) -# parser -parser = configargparse.ArgumentParser( - description="generate RST files from module recursively into /_gen", - config_file_parser_class=configargparse.YAMLConfigFileParser, - formatter_class=configargparse.ArgumentDefaultsHelpFormatter, -) -parser.add_argument( - "--root", type=str, help="root module to generate docs" -) -parser.add_argument("--dst", type=str, help="destination path to generate RSTs") -parser.add_argument("--exclude", nargs="*", default=[], help="exclude module name") -args = parser.parse_args() -print(args) - - -gendir = args.dst -os.makedirs(gendir, exist_ok=True) -os.makedirs(f"{gendir}/{args.root}", exist_ok=True) - -for p in glob(args.root + "/**", recursive=True): - module_name = to_module(p) - if any([ex in module_name for ex in args.exclude]): - continue - if "__init__" in p: - continue - if not p.endswith(".py"): - continue - - submodule_name = module_name.split(".")[1] - os.makedirs(f"{gendir}/{args.root}/{submodule_name}", exist_ok=True) - - if not os.path.exists(f"{gendir}/{args.root}/{submodule_name}/README.rst"): - # 1 get functions - for func in top_level_functions(parse_ast(p).body): - function_name = func.name - print(f"[INFO] generating {func.name} in {module_name}") - # 1.2 generate RST - with open(f"{gendir}/{args.root}/{submodule_name}/{function_name}.rst", "w") as f_rst: - gen_func_rst(f"{module_name}.{function_name}", f_rst, p, func.lineno) - - # 2 get classes - for clz in top_level_classes(parse_ast(p).body): - class_name = clz.name - print(f"[INFO] generating {clz.name} in {module_name}") - # 1.2 generate RST - with open(f"{gendir}/{args.root}/{submodule_name}/{class_name}.rst", "w") as f_rst: - gen_class_rst(f"{module_name}.{class_name}", f_rst, p, clz.lineno) + +if __name__ == "__main__": + # parser + parser = configargparse.ArgumentParser( + description="generate RST files from module recursively into /_gen", + config_file_parser_class=configargparse.YAMLConfigFileParser, + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--root", type=str, help="root module to generate docs" + ) + parser.add_argument("--dst", type=str, help="destination path to generate RSTs") + parser.add_argument("--exclude", nargs="*", default=[], help="exclude module name") + args = parser.parse_args() + print(args) + + + gendir = args.dst + os.makedirs(gendir, exist_ok=True) + os.makedirs(f"{gendir}/{args.root}", exist_ok=True) + + for p in glob(args.root + "/**", recursive=True): + module_name = to_module(p) + if any([ex in module_name for ex in args.exclude]): + continue + if "__init__" in p: + continue + if not p.endswith(".py"): + continue + + submodule_name = module_name.split(".")[1] + os.makedirs(f"{gendir}/{args.root}/{submodule_name}", exist_ok=True) + + if not os.path.exists(f"{gendir}/{args.root}/{submodule_name}/README.rst"): + # 1 get functions + for func in top_level_functions(parse_ast(p).body): + function_name = func.name + print(f"[INFO] generating {func.name} in {module_name}") + # 1.2 generate RST + with open(f"{gendir}/{args.root}/{submodule_name}/{function_name}.rst", "w") as f_rst: + gen_func_rst(f"{module_name}.{function_name}", f_rst, p, func.lineno) + + # 2 get classes + for clz in top_level_classes(parse_ast(p).body): + class_name = clz.name + print(f"[INFO] generating {clz.name} in {module_name}") + # 1.2 generate RST + with open(f"{gendir}/{args.root}/{submodule_name}/{class_name}.rst", "w") as f_rst: + gen_class_rst(f"{module_name}.{class_name}", f_rst, p, clz.lineno)