From acbd55ded2edb98ba538f7eda70f07b821c21fad Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 10 Nov 2023 23:01:56 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20=E6=8F=92=E4=BB=B6=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E6=94=B9=E4=B8=BA=E7=9B=B4=E6=8E=A5=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=BA=90=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/plugin/host.py | 94 +++++++++++++++++++++++++++++++++------ tests/repo_regexp_test.py | 7 +++ 2 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 tests/repo_regexp_test.py diff --git a/pkg/plugin/host.py b/pkg/plugin/host.py index d0fcf1f2..42643e2f 100644 --- a/pkg/plugin/host.py +++ b/pkg/plugin/host.py @@ -7,14 +7,17 @@ import sys import shutil import traceback +import re import pkg.utils.updater as updater import pkg.utils.context as context import pkg.plugin.switch as switch import pkg.plugin.settings as settings import pkg.qqbot.adapter as msadapter +import pkg.utils.network as network from mirai import Mirai +import requests from CallingGPT.session.session import Session @@ -155,29 +158,92 @@ def unload_plugins(): # logging.error("插件{}卸载时发生错误: {}".format(plugin['name'], sys.exc_info())) +def get_github_plugin_repo_label(repo_url: str) -> list[str]: + """获取username, repo""" + + # 提取 username/repo , 正则表达式 + repo = re.findall(r'(?:https?://github\.com/|git@github\.com:)([^/]+/[^/]+?)(?:\.git|/|$)', repo_url) + + if len(repo) > 0: # github + return repo[0].split("/") + else: + return None + + +def download_plugin_source_coder(repo_url: str, target_path: str): + """下载插件源码""" + # 检查源类型 + + # 提取 username/repo , 正则表达式 + repo = get_github_plugin_repo_label(repo_url) + + if repo is not None: # github + logging.info("从 GitHub 下载插件源码...") + + zipball_url = f"https://api.github.com/repos/{'/'.join(repo)}/zipball/HEAD" + + zip_resp = requests.get( + url=zipball_url, + proxies=network.wrapper_proxies(), + stream=True + ) + + if zip_resp.status_code != 200: + raise Exception("下载源码失败: {}".format(zip_resp.text)) + + if os.path.exists("temp/"+target_path): + shutil.rmtree("temp/"+target_path) + + if os.path.exists(target_path): + shutil.rmtree(target_path) + + os.makedirs("temp/"+target_path) + + with open("temp/"+target_path+"/source.zip", "wb") as f: + for chunk in zip_resp.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + + logging.info("下载完成, 解压...") + import zipfile + with zipfile.ZipFile("temp/"+target_path+"/source.zip", 'r') as zip_ref: + zip_ref.extractall("temp/"+target_path) + os.remove("temp/"+target_path+"/source.zip") + + # 目标是 username-repo-hash , 用正则表达式提取完整的文件夹名，复制到 plugins/repo + import glob + + # 获取解压后的文件夹名 + unzip_dir = glob.glob("temp/"+target_path+"/*")[0] + + # 复制到 plugins/repo + shutil.copytree(unzip_dir, target_path+"/") + + # 删除解压后的文件夹 + shutil.rmtree(unzip_dir) + + logging.info("解压完成") + else: + raise Exception("暂不支持的源类型，请使用 GitHub 仓库发行插件。") + + def install_plugin(repo_url: str): """安装插件，从git储存库获取并解决依赖""" - try: - import pkg.utils.pkgmgr - pkg.utils.pkgmgr.ensure_dulwich() - except: - pass - try: - import dulwich - except ModuleNotFoundError: - raise Exception("dulwich模块未安装,请查看 https://github.com/RockChinQ/QChatGPT/issues/77") + repo_label = None - from dulwich import porcelain + repo_label = get_github_plugin_repo_label(repo_url) + + if repo_label is None: + raise Exception("暂不支持的源类型，请使用 GitHub 仓库发行插件。") - logging.info("克隆插件储存库: {}".format(repo_url)) - repo = porcelain.clone(repo_url, "plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/", checkout=True) + download_plugin_source_coder(repo_url, "plugins/"+repo_label[1]) # 检查此目录是否包含requirements.txt - if os.path.exists("plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/requirements.txt"): + if os.path.exists("plugins/"+repo_label[1]+"/requirements.txt"): logging.info("检测到requirements.txt，正在安装依赖") import pkg.utils.pkgmgr - pkg.utils.pkgmgr.install_requirements("plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/requirements.txt") + pkg.utils.pkgmgr.install_requirements("plugins/"+repo_label[1]+"/requirements.txt") import pkg.utils.log as log log.reset_logging() diff --git a/tests/repo_regexp_test.py b/tests/repo_regexp_test.py new file mode 100644 index 00000000..5bf78f9d --- /dev/null +++ b/tests/repo_regexp_test.py @@ -0,0 +1,7 @@ +import re + +repo_url = "git@github.com:RockChinQ/WebwlkrPlugin.git" + +repo = re.findall(r'(?:https?://github\.com/|git@github\.com:)([^/]+/[^/]+?)(?:\.git|/|$)', repo_url) + +print(repo) \ No newline at end of file From 67a208bc9054175902304710173f39f64b1a7713 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 11 Nov 2023 17:38:52 +0800 Subject: [PATCH 2/7] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=85=83=E6=95=B0=E6=8D=AE=E6=93=8D=E4=BD=9C=E6=A8=A1?= =?UTF-8?q?=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/plugin/metadata.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pkg/plugin/metadata.py diff --git a/pkg/plugin/metadata.py b/pkg/plugin/metadata.py new file mode 100644 index 00000000..e69de29b From 76d7db88ea4c4ead56a6104ed558fbf9b17afe31 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sat, 11 Nov 2023 23:17:28 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat:=20=E5=9F=BA=E4=BA=8E=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=AE=B0=E5=BD=95=E7=9A=84=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/plugin/host.py | 86 ++++++++++++++++++-------------- pkg/plugin/metadata.py | 87 +++++++++++++++++++++++++++++++++ pkg/qqbot/cmds/plugin/plugin.py | 8 +-- 3 files changed, 142 insertions(+), 39 deletions(-) diff --git a/pkg/plugin/host.py b/pkg/plugin/host.py index 42643e2f..db4da878 100644 --- a/pkg/plugin/host.py +++ b/pkg/plugin/host.py @@ -7,6 +7,7 @@ import sys import shutil import traceback +import time import re import pkg.utils.updater as updater @@ -15,6 +16,7 @@ import pkg.plugin.settings as settings import pkg.qqbot.adapter as msadapter import pkg.utils.network as network +import pkg.plugin.metadata as metadata from mirai import Mirai import requests @@ -116,10 +118,15 @@ def load_plugins(): # 加载插件顺序 settings.load_settings() + logging.debug("registered plugins: {}".format(__plugins__)) + # 输出已注册的内容函数列表 logging.debug("registered content functions: {}".format(__callable_functions__)) logging.debug("function instance map: {}".format(__function_inst_map__)) + # 迁移插件源地址记录 + metadata.do_plugin_git_repo_migrate() + def initialize_plugins(): """初始化插件""" @@ -170,13 +177,15 @@ def get_github_plugin_repo_label(repo_url: str) -> list[str]: return None -def download_plugin_source_coder(repo_url: str, target_path: str): +def download_plugin_source_code(repo_url: str, target_path: str) -> str: """下载插件源码""" # 检查源类型 # 提取 username/repo , 正则表达式 repo = get_github_plugin_repo_label(repo_url) + target_path += repo[1] + if repo is not None: # github logging.info("从 GitHub 下载插件源码...") @@ -225,30 +234,31 @@ def download_plugin_source_coder(repo_url: str, target_path: str): logging.info("解压完成") else: raise Exception("暂不支持的源类型，请使用 GitHub 仓库发行插件。") + + return repo[1] -def install_plugin(repo_url: str): - """安装插件，从git储存库获取并解决依赖""" - - repo_label = None - - repo_label = get_github_plugin_repo_label(repo_url) - - if repo_label is None: - raise Exception("暂不支持的源类型，请使用 GitHub 仓库发行插件。") - - download_plugin_source_coder(repo_url, "plugins/"+repo_label[1]) - +def check_requirements(path: str): # 检查此目录是否包含requirements.txt - if os.path.exists("plugins/"+repo_label[1]+"/requirements.txt"): + if os.path.exists(path+"/requirements.txt"): logging.info("检测到requirements.txt，正在安装依赖") import pkg.utils.pkgmgr - pkg.utils.pkgmgr.install_requirements("plugins/"+repo_label[1]+"/requirements.txt") + pkg.utils.pkgmgr.install_requirements(path+"/requirements.txt") import pkg.utils.log as log log.reset_logging() +def install_plugin(repo_url: str): + """安装插件，从git储存库获取并解决依赖""" + + repo_label = download_plugin_source_code(repo_url, "plugins/") + + check_requirements("plugins/"+repo_label) + + metadata.set_plugin_metadata(repo_label, repo_url, int(time.time()), "HEAD") + + def uninstall_plugin(plugin_name: str) -> str: """卸载插件""" if plugin_name not in __plugins__: @@ -268,39 +278,43 @@ def uninstall_plugin(plugin_name: str) -> str: def update_plugin(plugin_name: str): """更新插件""" # 检查是否有远程地址记录 - target_plugin_dir = "plugins/" + __plugins__[plugin_name]['path'].replace("\\", "/").split("plugins/")[1].split("/")[0] + plugin_path_name = get_plugin_path_name_by_plugin_name(plugin_name) + + meta = metadata.get_plugin_metadata(plugin_path_name) - remote_url = updater.get_remote_url(target_plugin_dir) + if meta == {}: + raise Exception("没有此插件元数据信息，无法更新") + + remote_url = meta['source'] if remote_url == "https://github.com/RockChinQ/QChatGPT" or remote_url == "https://gitee.com/RockChin/QChatGPT" \ or remote_url == "" or remote_url is None or remote_url == "http://github.com/RockChinQ/QChatGPT" or remote_url == "http://gitee.com/RockChin/QChatGPT": raise Exception("插件没有远程地址记录，无法更新") - # 把远程clone到temp/plugins/update/插件名 - logging.info("克隆插件储存库: {}".format(remote_url)) + # 重新安装插件 + logging.info("正在重新安装插件以进行更新...") - from dulwich import porcelain - clone_target_dir = "temp/plugins/update/"+target_plugin_dir.split("/")[-1]+"/" + install_plugin(remote_url) - if os.path.exists(clone_target_dir): - shutil.rmtree(clone_target_dir) - if not os.path.exists(clone_target_dir): - os.makedirs(clone_target_dir) - repo = porcelain.clone(remote_url, clone_target_dir, checkout=True) +def get_plugin_name_by_path_name(plugin_path_name: str) -> str: + for k, v in __plugins__.items(): + if v['path'] == "plugins/"+plugin_path_name+"/main.py": + return k + return None - # 检查此目录是否包含requirements.txt - if os.path.exists(clone_target_dir+"requirements.txt"): - logging.info("检测到requirements.txt，正在安装依赖") - import pkg.utils.pkgmgr - pkg.utils.pkgmgr.install_requirements(clone_target_dir+"requirements.txt") - import pkg.utils.log as log - log.reset_logging() +def get_plugin_path_name_by_plugin_name(plugin_name: str) -> str: + if plugin_name not in __plugins__: + return None + + plugin_main_module_path = __plugins__[plugin_name]['path'] + + plugin_main_module_path = plugin_main_module_path.replace("\\", "/") + + spt = plugin_main_module_path.split("/") - # 将temp/plugins/update/插件名 覆盖到 plugins/插件名 - shutil.rmtree(target_plugin_dir) + return spt[1] - shutil.copytree(clone_target_dir, target_plugin_dir) class EventContext: """事件上下文""" diff --git a/pkg/plugin/metadata.py b/pkg/plugin/metadata.py index e69de29b..51de742e 100644 --- a/pkg/plugin/metadata.py +++ b/pkg/plugin/metadata.py @@ -0,0 +1,87 @@ +import os +import shutil +import json +import time + +import dulwich.errors as dulwich_err + +from ..utils import updater + + +def read_metadata_file() -> dict: + # 读取 plugins/metadata.json 文件 + if not os.path.exists('plugins/metadata.json'): + return {} + with open('plugins/metadata.json', 'r') as f: + return json.load(f) + + +def write_metadata_file(metadata: dict): + if not os.path.exists('plugins'): + os.mkdir('plugins') + + with open('plugins/metadata.json', 'w') as f: + json.dump(metadata, f, indent=4, ensure_ascii=False) + + +def do_plugin_git_repo_migrate(): + # 仅在 plugins/metadata.json 不存在时执行 + if os.path.exists('plugins/metadata.json'): + return + + metadata = read_metadata_file() + + # 遍历 plugins 下所有目录，获取目录的git远程地址 + for plugin_name in os.listdir('plugins'): + plugin_path = os.path.join('plugins', plugin_name) + if not os.path.isdir(plugin_path): + continue + + remote_url = None + try: + remote_url = updater.get_remote_url(plugin_path) + except dulwich_err.NotGitRepository: + continue + if remote_url == "https://github.com/RockChinQ/QChatGPT" or remote_url == "https://gitee.com/RockChin/QChatGPT" \ + or remote_url == "" or remote_url is None or remote_url == "http://github.com/RockChinQ/QChatGPT" or remote_url == "http://gitee.com/RockChin/QChatGPT": + continue + + from . import host + + if plugin_name not in metadata: + metadata[plugin_name] = { + 'source': remote_url, + 'install_timestamp': int(time.time()), + 'ref': 'HEAD', + } + + write_metadata_file(metadata) + + +def set_plugin_metadata( + plugin_name: str, + source: str, + install_timestamp: int, + ref: str, +): + metadata = read_metadata_file() + metadata[plugin_name] = { + 'source': source, + 'install_timestamp': install_timestamp, + 'ref': ref, + } + write_metadata_file(metadata) + + +def remove_plugin_metadata(plugin_name: str): + metadata = read_metadata_file() + if plugin_name in metadata: + del metadata[plugin_name] + write_metadata_file(metadata) + + +def get_plugin_metadata(plugin_name: str) -> dict: + metadata = read_metadata_file() + if plugin_name in metadata: + return metadata[plugin_name] + return {} \ No newline at end of file diff --git a/pkg/qqbot/cmds/plugin/plugin.py b/pkg/qqbot/cmds/plugin/plugin.py index 9818b7c7..b0bbebf8 100644 --- a/pkg/qqbot/cmds/plugin/plugin.py +++ b/pkg/qqbot/cmds/plugin/plugin.py @@ -84,7 +84,7 @@ def closure(): @AbstractCommandNode.register( parent=PluginCommand, name="update", - description="更新所有插件", + description="更新插件", usage="!plugin update", aliases=[], privilege=2 @@ -110,7 +110,9 @@ def closure(): plugin_host.update_plugin(key) updated.append(key) else: - if ctx.crt_params[0] in plugin_list: + plugin_path_name = plugin_host.get_plugin_path_name_by_plugin_name(ctx.crt_params[0]) + + if ctx.crt_params[0] is not None: plugin_host.update_plugin(ctx.crt_params[0]) updated.append(ctx.crt_params[0]) else: @@ -119,7 +121,7 @@ def closure(): pkg.utils.context.get_qqbot_manager().notify_admin("已更新插件: {}, 请发送 !reload 重载插件".format(", ".join(updated))) except Exception as e: logging.error("插件更新失败:{}".format(e)) - pkg.utils.context.get_qqbot_manager().notify_admin("插件更新失败:{} 请尝试手动更新插件".format(e)) + pkg.utils.context.get_qqbot_manager().notify_admin("插件更新失败:{} 请使用 !plugin 命令确认插件名称或尝试手动更新插件".format(e)) reply = ["[bot]正在更新插件，请勿重复发起..."] threading.Thread(target=closure).start() From 89c83ebf201f8a530e8993e5fc4cc0dd3db1d858 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sun, 12 Nov 2023 11:30:10 +0800 Subject: [PATCH 4/7] =?UTF-8?q?fix:=20=E9=94=99=E8=AF=AF=E7=9A=84=E5=88=A4?= =?UTF-8?q?=E7=A9=BA=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/qqbot/cmds/plugin/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/qqbot/cmds/plugin/plugin.py b/pkg/qqbot/cmds/plugin/plugin.py index b0bbebf8..093ebe77 100644 --- a/pkg/qqbot/cmds/plugin/plugin.py +++ b/pkg/qqbot/cmds/plugin/plugin.py @@ -112,7 +112,7 @@ def closure(): else: plugin_path_name = plugin_host.get_plugin_path_name_by_plugin_name(ctx.crt_params[0]) - if ctx.crt_params[0] is not None: + if plugin_path_name is not None: plugin_host.update_plugin(ctx.crt_params[0]) updated.append(ctx.crt_params[0]) else: From 1419d7611da67eacc68ecfb544f11363f564e026 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sun, 12 Nov 2023 12:03:52 +0800 Subject: [PATCH 5/7] =?UTF-8?q?ci(cmdpriv):=20=E6=9C=AC=E5=9C=B0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E9=80=9A=E8=BF=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/update-cmdpriv-template.yml | 6 +++--- .github/workflows/update-override-all.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-cmdpriv-template.yml b/.github/workflows/update-cmdpriv-template.yml index 522f03b8..1af42cd8 100644 --- a/.github/workflows/update-cmdpriv-template.yml +++ b/.github/workflows/update-cmdpriv-template.yml @@ -21,12 +21,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.x + python-version: 3.10.13 - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install --upgrade yiri-mirai openai colorlog func_timeout dulwich Pillow CallingGPT tiktoken + python -m pip install --upgrade yiri-mirai openai>=1.0.0 colorlog func_timeout dulwich Pillow CallingGPT tiktoken + python -m pip install -U openai>=1.0.0 - name: Copy Scripts run: | diff --git a/.github/workflows/update-override-all.yml b/.github/workflows/update-override-all.yml index ec3efdbb..83ef6a6b 100644 --- a/.github/workflows/update-override-all.yml +++ b/.github/workflows/update-override-all.yml @@ -29,7 +29,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # 在此处添加您的项目所需的其他依赖 - name: Copy Scripts run: | From a8b1e6ce9144d0d1ebfb48925ad5ea15d92a6963 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sun, 12 Nov 2023 12:05:04 +0800 Subject: [PATCH 6/7] ci: test --- pkg/qqbot/cmds/plugin/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/qqbot/cmds/plugin/plugin.py b/pkg/qqbot/cmds/plugin/plugin.py index 093ebe77..00ba44f0 100644 --- a/pkg/qqbot/cmds/plugin/plugin.py +++ b/pkg/qqbot/cmds/plugin/plugin.py @@ -84,7 +84,7 @@ def closure(): @AbstractCommandNode.register( parent=PluginCommand, name="update", - description="更新插件", + description="更新指定插件或全部插件", usage="!plugin update", aliases=[], privilege=2 From 71b8bf13e44fc9aa94edca38cf0510e1f3468f7b Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Sun, 12 Nov 2023 13:52:04 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix:=20=E6=8F=92=E4=BB=B6=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/plugin/host.py | 2 ++ pkg/qqbot/sources/nakuru.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/host.py b/pkg/plugin/host.py index db4da878..27a6c1e2 100644 --- a/pkg/plugin/host.py +++ b/pkg/plugin/host.py @@ -70,6 +70,8 @@ def generate_plugin_order(): def iter_plugins(): """按照顺序迭代插件""" for plugin_name in __plugins_order__: + if plugin_name not in __plugins__: + continue yield __plugins__[plugin_name] diff --git a/pkg/qqbot/sources/nakuru.py b/pkg/qqbot/sources/nakuru.py index 42102963..3f70b4b8 100644 --- a/pkg/qqbot/sources/nakuru.py +++ b/pkg/qqbot/sources/nakuru.py @@ -185,7 +185,11 @@ def __init__(self, cfg: dict): if resp.status_code == 403: logging.error("go-cqhttp拒绝访问，请检查config.py中nakuru_config的token是否与go-cqhttp设置的access-token匹配") raise Exception("go-cqhttp拒绝访问，请检查config.py中nakuru_config的token是否与go-cqhttp设置的access-token匹配") - self.bot_account_id = int(resp.json()['data']['user_id']) + try: + self.bot_account_id = int(resp.json()['data']['user_id']) + except Exception as e: + logging.error("获取go-cqhttp账号信息失败: {}, 请检查是否已启动go-cqhttp并配置正确".format(e)) + raise Exception("获取go-cqhttp账号信息失败: {}, 请检查是否已启动go-cqhttp并配置正确".format(e)) def send_message( self,