Merge pull request #592 from RockChinQ/fix/plugin-downloading

Feat: 通过 GitHub API 进行插件安装和更新

.github/workflows/update-cmdpriv-template.yml

@@ -21,12 +21,12 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.x
+        python-version: 3.10.13
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install --upgrade  yiri-mirai openai colorlog func_timeout dulwich Pillow CallingGPT tiktoken
+        python -m pip install --upgrade  yiri-mirai openai>=1.0.0 colorlog func_timeout dulwich Pillow CallingGPT tiktoken
+        python -m pip install -U openai>=1.0.0
 
     - name: Copy Scripts
       run: |

.github/workflows/update-override-all.yml

@@ -29,7 +29,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          # 在此处添加您的项目所需的其他依赖
 
       - name: Copy Scripts
         run: |

pkg/plugin/host.py

@@ -7,14 +7,19 @@
 import sys
 import shutil
 import traceback
+import time
+import re
 
 import pkg.utils.updater as updater
 import pkg.utils.context as context
 import pkg.plugin.switch as switch
 import pkg.plugin.settings as settings
 import pkg.qqbot.adapter as msadapter
+import pkg.utils.network as network
+import pkg.plugin.metadata as metadata
 
 from mirai import Mirai
+import requests
 
 from CallingGPT.session.session import Session
 
@@ -65,6 +70,8 @@ def generate_plugin_order():
 def iter_plugins():
     """按照顺序迭代插件"""
     for plugin_name in __plugins_order__:
+        if plugin_name not in __plugins__:
+            continue
         yield __plugins__[plugin_name]
 
 
@@ -113,10 +120,15 @@ def load_plugins():
     # 加载插件顺序
     settings.load_settings()
 
+    logging.debug("registered plugins: {}".format(__plugins__))
+
     # 输出已注册的内容函数列表
     logging.debug("registered content functions: {}".format(__callable_functions__))
     logging.debug("function instance map: {}".format(__function_inst_map__))
 
+    # 迁移插件源地址记录
+    metadata.do_plugin_git_repo_migrate()
+
 
 def initialize_plugins():
     """初始化插件"""
@@ -155,34 +167,100 @@ def unload_plugins():
     #                 logging.error("插件{}卸载时发生错误: {}".format(plugin['name'], sys.exc_info()))
 
 
-def install_plugin(repo_url: str):
-    """安装插件，从git储存库获取并解决依赖"""
-    try:
-        import pkg.utils.pkgmgr
-        pkg.utils.pkgmgr.ensure_dulwich()
-    except:
-        pass
+def get_github_plugin_repo_label(repo_url: str) -> list[str]:
+    """获取username, repo"""
+
+    # 提取 username/repo , 正则表达式
+    repo =  re.findall(r'(?:https?://github\.com/|git@github\.com:)([^/]+/[^/]+?)(?:\.git|/|$)', repo_url)
+
+    if len(repo) > 0: # github
+        return repo[0].split("/")
+    else:
+        return None
+
+
+def download_plugin_source_code(repo_url: str, target_path: str) -> str:
+    """下载插件源码"""
+    # 检查源类型
+
+    # 提取 username/repo , 正则表达式
+    repo =  get_github_plugin_repo_label(repo_url)
 
-    try:
-        import dulwich
-    except ModuleNotFoundError:
-        raise Exception("dulwich模块未安装,请查看 https://github.com/RockChinQ/QChatGPT/issues/77")
+    target_path += repo[1]
 
-    from dulwich import porcelain
+    if repo is not None: # github
+        logging.info("从 GitHub 下载插件源码...")
 
-    logging.info("克隆插件储存库: {}".format(repo_url))
-    repo = porcelain.clone(repo_url, "plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/", checkout=True)
+        zipball_url = f"https://api.github.com/repos/{'/'.join(repo)}/zipball/HEAD"
 
+        zip_resp = requests.get(
+            url=zipball_url,
+            proxies=network.wrapper_proxies(),
+            stream=True
+        )
+
+        if zip_resp.status_code != 200:
+            raise Exception("下载源码失败: {}".format(zip_resp.text))
+
+        if os.path.exists("temp/"+target_path):
+            shutil.rmtree("temp/"+target_path)
+
+        if os.path.exists(target_path):
+            shutil.rmtree(target_path)
+
+        os.makedirs("temp/"+target_path)
+
+        with open("temp/"+target_path+"/source.zip", "wb") as f:
+            for chunk in zip_resp.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+
+        logging.info("下载完成, 解压...")
+        import zipfile
+        with zipfile.ZipFile("temp/"+target_path+"/source.zip", 'r') as zip_ref:
+            zip_ref.extractall("temp/"+target_path)
+        os.remove("temp/"+target_path+"/source.zip")
+
+        # 目标是 username-repo-hash , 用正则表达式提取完整的文件夹名，复制到 plugins/repo
+        import glob
+
+        # 获取解压后的文件夹名
+        unzip_dir = glob.glob("temp/"+target_path+"/*")[0]
+
+        # 复制到 plugins/repo
+        shutil.copytree(unzip_dir, target_path+"/")
+
+        # 删除解压后的文件夹
+        shutil.rmtree(unzip_dir)
+
+        logging.info("解压完成")
+    else:
+        raise Exception("暂不支持的源类型，请使用 GitHub 仓库发行插件。")
+
+    return repo[1]
+
+
+def check_requirements(path: str):
     # 检查此目录是否包含requirements.txt
-    if os.path.exists("plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/requirements.txt"):
+    if os.path.exists(path+"/requirements.txt"):
         logging.info("检测到requirements.txt，正在安装依赖")
         import pkg.utils.pkgmgr
-        pkg.utils.pkgmgr.install_requirements("plugins/"+repo_url.split(".git")[0].split("/")[-1]+"/requirements.txt")
+        pkg.utils.pkgmgr.install_requirements(path+"/requirements.txt")
 
         import pkg.utils.log as log
         log.reset_logging()
 
 
+def install_plugin(repo_url: str):
+    """安装插件，从git储存库获取并解决依赖"""
+
+    repo_label = download_plugin_source_code(repo_url, "plugins/")
+
+    check_requirements("plugins/"+repo_label)
+
+    metadata.set_plugin_metadata(repo_label, repo_url, int(time.time()), "HEAD")
+
+
 def uninstall_plugin(plugin_name: str) -> str:
     """卸载插件"""
     if plugin_name not in __plugins__:
@@ -202,39 +280,43 @@ def uninstall_plugin(plugin_name: str) -> str:
 def update_plugin(plugin_name: str):
     """更新插件"""
     # 检查是否有远程地址记录
-    target_plugin_dir = "plugins/" + __plugins__[plugin_name]['path'].replace("\\", "/").split("plugins/")[1].split("/")[0]
+    plugin_path_name = get_plugin_path_name_by_plugin_name(plugin_name)
+
+    meta = metadata.get_plugin_metadata(plugin_path_name)
+
+    if meta == {}:
+        raise Exception("没有此插件元数据信息，无法更新")
 
-    remote_url = updater.get_remote_url(target_plugin_dir)
+    remote_url = meta['source']
     if remote_url == "https://github.com/RockChinQ/QChatGPT" or remote_url == "https://gitee.com/RockChin/QChatGPT" \
         or remote_url == "" or remote_url is None or remote_url == "http://github.com/RockChinQ/QChatGPT" or remote_url == "http://gitee.com/RockChin/QChatGPT":
         raise Exception("插件没有远程地址记录，无法更新")
 
-    # 把远程clone到temp/plugins/update/插件名
-    logging.info("克隆插件储存库: {}".format(remote_url))
+    # 重新安装插件
+    logging.info("正在重新安装插件以进行更新...")
 
-    from dulwich import porcelain
-    clone_target_dir = "temp/plugins/update/"+target_plugin_dir.split("/")[-1]+"/"
+    install_plugin(remote_url)
 
-    if os.path.exists(clone_target_dir):
-        shutil.rmtree(clone_target_dir)
 
-    if not os.path.exists(clone_target_dir):
-        os.makedirs(clone_target_dir)
-    repo = porcelain.clone(remote_url, clone_target_dir, checkout=True)
+def get_plugin_name_by_path_name(plugin_path_name: str) -> str:
+    for k, v in __plugins__.items():
+        if v['path'] == "plugins/"+plugin_path_name+"/main.py":
+            return k
+    return None
 
-    # 检查此目录是否包含requirements.txt
-    if os.path.exists(clone_target_dir+"requirements.txt"):
-        logging.info("检测到requirements.txt，正在安装依赖")
-        import pkg.utils.pkgmgr
-        pkg.utils.pkgmgr.install_requirements(clone_target_dir+"requirements.txt")
 
-        import pkg.utils.log as log
-        log.reset_logging()
+def get_plugin_path_name_by_plugin_name(plugin_name: str) -> str:
+    if plugin_name not in __plugins__:
+        return None
+
+    plugin_main_module_path = __plugins__[plugin_name]['path']
+
+    plugin_main_module_path = plugin_main_module_path.replace("\\", "/")
+
+    spt = plugin_main_module_path.split("/")
 
-    # 将temp/plugins/update/插件名 覆盖到 plugins/插件名
-    shutil.rmtree(target_plugin_dir)
+    return spt[1]
 
-    shutil.copytree(clone_target_dir, target_plugin_dir)
 
 class EventContext:
     """事件上下文"""

pkg/plugin/metadata.py

@@ -0,0 +1,87 @@
+import os
+import shutil
+import json
+import time
+
+import dulwich.errors as dulwich_err
+
+from ..utils import updater
+
+
+def read_metadata_file() -> dict:
+    # 读取 plugins/metadata.json 文件
+    if not os.path.exists('plugins/metadata.json'):
+        return {}
+    with open('plugins/metadata.json', 'r') as f:
+        return json.load(f)
+
+
+def write_metadata_file(metadata: dict):
+    if not os.path.exists('plugins'):
+        os.mkdir('plugins')
+
+    with open('plugins/metadata.json', 'w') as f:
+        json.dump(metadata, f, indent=4, ensure_ascii=False)
+
+
+def do_plugin_git_repo_migrate():
+    # 仅在 plugins/metadata.json 不存在时执行
+    if os.path.exists('plugins/metadata.json'):
+        return
+
+    metadata = read_metadata_file()
+
+    # 遍历 plugins 下所有目录，获取目录的git远程地址
+    for plugin_name in os.listdir('plugins'):
+        plugin_path = os.path.join('plugins', plugin_name)
+        if not os.path.isdir(plugin_path):
+            continue
+
+        remote_url = None
+        try:
+            remote_url = updater.get_remote_url(plugin_path)
+        except dulwich_err.NotGitRepository:
+            continue
+        if remote_url == "https://github.com/RockChinQ/QChatGPT" or remote_url == "https://gitee.com/RockChin/QChatGPT" \
+            or remote_url == "" or remote_url is None or remote_url == "http://github.com/RockChinQ/QChatGPT" or remote_url == "http://gitee.com/RockChin/QChatGPT":
+            continue
+
+        from . import host
+
+        if plugin_name not in metadata:
+            metadata[plugin_name] = {
+                'source': remote_url,
+                'install_timestamp': int(time.time()),
+                'ref': 'HEAD',
+            }
+
+    write_metadata_file(metadata)
+
+
+def set_plugin_metadata(
+    plugin_name: str,
+    source: str,
+    install_timestamp: int,
+    ref: str,
+):
+    metadata = read_metadata_file()
+    metadata[plugin_name] = {
+        'source': source,
+        'install_timestamp': install_timestamp,
+        'ref': ref,
+    }
+    write_metadata_file(metadata)
+
+
+def remove_plugin_metadata(plugin_name: str):
+    metadata = read_metadata_file()
+    if plugin_name in metadata:
+        del metadata[plugin_name]
+        write_metadata_file(metadata)
+
+
+def get_plugin_metadata(plugin_name: str) -> dict:
+    metadata = read_metadata_file()
+    if plugin_name in metadata:
+        return metadata[plugin_name]
+    return {}

pkg/qqbot/cmds/plugin/plugin.py

@@ -84,7 +84,7 @@ def closure():
 @AbstractCommandNode.register(
     parent=PluginCommand,
     name="update",
-    description="更新所有插件",
+    description="更新指定插件或全部插件",
     usage="!plugin update",
     aliases=[],
     privilege=2
@@ -110,7 +110,9 @@ def closure():
                             plugin_host.update_plugin(key)
                             updated.append(key)
                     else:
-                        if ctx.crt_params[0] in plugin_list:
+                        plugin_path_name = plugin_host.get_plugin_path_name_by_plugin_name(ctx.crt_params[0])
+
+                        if plugin_path_name is not None:
                             plugin_host.update_plugin(ctx.crt_params[0])
                             updated.append(ctx.crt_params[0])
                         else:
@@ -119,7 +121,7 @@ def closure():
                     pkg.utils.context.get_qqbot_manager().notify_admin("已更新插件: {}, 请发送 !reload 重载插件".format(", ".join(updated)))
                 except Exception as e:
                     logging.error("插件更新失败:{}".format(e))
-                    pkg.utils.context.get_qqbot_manager().notify_admin("插件更新失败:{} 请尝试手动更新插件".format(e))
+                    pkg.utils.context.get_qqbot_manager().notify_admin("插件更新失败:{} 请使用 !plugin 命令确认插件名称或尝试手动更新插件".format(e))
 
             reply = ["[bot]正在更新插件，请勿重复发起..."]
             threading.Thread(target=closure).start()

pkg/qqbot/sources/nakuru.py

@@ -185,7 +185,11 @@ def __init__(self, cfg: dict):
         if resp.status_code == 403:
             logging.error("go-cqhttp拒绝访问，请检查config.py中nakuru_config的token是否与go-cqhttp设置的access-token匹配")
             raise Exception("go-cqhttp拒绝访问，请检查config.py中nakuru_config的token是否与go-cqhttp设置的access-token匹配")
-        self.bot_account_id = int(resp.json()['data']['user_id'])
+        try:
+            self.bot_account_id = int(resp.json()['data']['user_id'])
+        except Exception as e:
+            logging.error("获取go-cqhttp账号信息失败: {}, 请检查是否已启动go-cqhttp并配置正确".format(e))
+            raise Exception("获取go-cqhttp账号信息失败: {}, 请检查是否已启动go-cqhttp并配置正确".format(e))
 
     def send_message(
         self,

tests/repo_regexp_test.py

@@ -0,0 +1,7 @@
+import re
+
+repo_url = "[email protected]:RockChinQ/WebwlkrPlugin.git"
+
+repo =  re.findall(r'(?:https?://github\.com/|git@github\.com:)([^/]+/[^/]+?)(?:\.git|/|$)', repo_url)
+
+print(repo)