diff --git a/pkg/config/migration.py b/pkg/config/migration.py new file mode 100644 index 00000000..3a6650b2 --- /dev/null +++ b/pkg/config/migration.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import abc +import typing + +from ..core import app + + +preregistered_migrations: list[typing.Type[Migration]] = [] + +def migration_class(name: str, number: int): + """注册一个迁移 + """ + def decorator(cls: typing.Type[Migration]) -> typing.Type[Migration]: + cls.name = name + cls.number = number + preregistered_migrations.append(cls) + return cls + + return decorator + + +class Migration(abc.ABC): + """一个版本的迁移 + """ + + name: str + + number: int + + ap: app.Application + + def __init__(self, ap: app.Application): + self.ap = ap + + @abc.abstractmethod + async def need_migrate(self) -> bool: + """判断当前环境是否需要运行此迁移 + """ + pass + + @abc.abstractmethod + async def run(self): + """执行迁移 + """ + pass diff --git a/pkg/config/migrations/__init__.py b/pkg/config/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pkg/config/migrations/m1_sensitive_word_migration.py b/pkg/config/migrations/m1_sensitive_word_migration.py new file mode 100644 index 00000000..cb8fc1c6 --- /dev/null +++ b/pkg/config/migrations/m1_sensitive_word_migration.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import os +import sys + +from .. import migration + + +@migration.migration_class("sensitive-word-migration", 1) +class SensitiveWordMigration(migration.Migration): + """敏感词迁移 + """ + + async def need_migrate(self) -> bool: + """判断当前环境是否需要运行此迁移 + """ + return os.path.exists("data/config/sensitive-words.json") + + async def run(self): + """执行迁移 + """ + # 移动文件 + os.rename("data/config/sensitive-words.json", "data/metadata/sensitive-words.json") + + # 重新加载配置 + await self.ap.sensitive_meta.load_config() diff --git a/pkg/core/app.py b/pkg/core/app.py index 0d726a44..e1df54ec 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -33,6 +33,8 @@ class Application: tool_mgr: llm_tool_mgr.ToolManager = None + # ======= 配置管理器 ======= + command_cfg: config_mgr.ConfigManager = None pipeline_cfg: config_mgr.ConfigManager = None @@ -43,6 +45,16 @@ class Application: system_cfg: config_mgr.ConfigManager = None + # ======= 元数据配置管理器 ======= + + sensitive_meta: config_mgr.ConfigManager = None + + adapter_qq_botpy_meta: config_mgr.ConfigManager = None + + plugin_setting_meta: config_mgr.ConfigManager = None + + # ========================= + ctr_mgr: center_mgr.V2CenterAPI = None plugin_mgr: plugin_mgr.PluginManager = None diff --git a/pkg/core/boot.py b/pkg/core/boot.py index 46f06760..081a2f84 100644 --- a/pkg/core/boot.py +++ b/pkg/core/boot.py @@ -5,11 +5,12 @@ from . import stage # 引入启动阶段实现以便注册 -from .stages import load_config, setup_logger, build_app +from .stages import load_config, setup_logger, build_app, migrate stage_order = [ "LoadConfigStage", + "MigrationStage", "SetupLoggerStage", "BuildAppStage" ] diff --git a/pkg/core/bootutils/files.py b/pkg/core/bootutils/files.py index ab4b15cf..9bab3826 100644 --- a/pkg/core/bootutils/files.py +++ b/pkg/core/bootutils/files.py @@ -13,7 +13,6 @@ "data/config/platform.json": "templates/platform.json", "data/config/provider.json": "templates/provider.json", "data/config/system.json": "templates/system.json", - "data/config/sensitive-words.json": "templates/sensitive-words.json", "data/scenario/default.json": "templates/scenario-template.json", } diff --git a/pkg/core/stages/load_config.py b/pkg/core/stages/load_config.py index 0ee2c1e4..7f83dbad 100644 --- a/pkg/core/stages/load_config.py +++ b/pkg/core/stages/load_config.py @@ -17,3 +17,13 @@ async def run(self, ap: app.Application): ap.platform_cfg = await config.load_json_config("data/config/platform.json", "templates/platform.json") ap.provider_cfg = await config.load_json_config("data/config/provider.json", "templates/provider.json") ap.system_cfg = await config.load_json_config("data/config/system.json", "templates/system.json") + + + ap.plugin_setting_meta = await config.load_json_config("plugins/plugins.json", "templates/plugin-settings.json") + await ap.plugin_setting_meta.dump_config() + + ap.sensitive_meta = await config.load_json_config("data/metadata/sensitive-words.json", "templates/metadata/sensitive-words.json") + await ap.sensitive_meta.dump_config() + + ap.adapter_qq_botpy_meta = await config.load_json_config("data/metadata/adapter-qq-botpy.json", "templates/metadata/adapter-qq-botpy.json") + await ap.adapter_qq_botpy_meta.dump_config() diff --git a/pkg/core/stages/migrate.py b/pkg/core/stages/migrate.py new file mode 100644 index 00000000..5e84b182 --- /dev/null +++ b/pkg/core/stages/migrate.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import importlib + +from .. import stage, app +from ...config import migration +from ...config.migrations import m1_sensitive_word_migration + + +@stage.stage_class("MigrationStage") +class MigrationStage(stage.BootingStage): + """迁移阶段 + """ + + async def run(self, ap: app.Application): + """启动 + """ + + migrations = migration.preregistered_migrations + + # 按照迁移号排序 + migrations.sort(key=lambda x: x.number) + + for migration_cls in migrations: + migration_instance = migration_cls(ap) + + if await migration_instance.need_migrate(): + await migration_instance.run() diff --git a/pkg/pipeline/cntfilter/filters/banwords.py b/pkg/pipeline/cntfilter/filters/banwords.py index 9391971c..5cd7dcfa 100644 --- a/pkg/pipeline/cntfilter/filters/banwords.py +++ b/pkg/pipeline/cntfilter/filters/banwords.py @@ -10,31 +10,26 @@ class BanWordFilter(filter_model.ContentFilter): """根据内容禁言""" - sensitive: cfg_mgr.ConfigManager - async def initialize(self): - self.sensitive = await cfg_mgr.load_json_config( - "data/config/sensitive-words.json", - "templates/sensitive-words.json" - ) + pass async def process(self, message: str) -> entities.FilterResult: found = False - for word in self.sensitive.data['words']: + for word in self.ap.sensitive_meta.data['words']: match = re.findall(word, message) if len(match) > 0: found = True for i in range(len(match)): - if self.sensitive.data['mask_word'] == "": + if self.ap.sensitive_meta.data['mask_word'] == "": message = message.replace( - match[i], self.sensitive.data['mask'] * len(match[i]) + match[i], self.ap.sensitive_meta.data['mask'] * len(match[i]) ) else: message = message.replace( - match[i], self.sensitive.data['mask_word'] + match[i], self.ap.sensitive_meta.data['mask_word'] ) return entities.FilterResult( diff --git a/pkg/platform/sources/qqbotpy.py b/pkg/platform/sources/qqbotpy.py index ab981cfe..5d731d34 100644 --- a/pkg/platform/sources/qqbotpy.py +++ b/pkg/platform/sources/qqbotpy.py @@ -506,10 +506,7 @@ def unregister_listener( async def run_async(self): - self.metadata = await cfg_mgr.load_json_config( - "data/metadata/adapter-qq-botpy.json", - "templates/metadata/adapter-qq-botpy.json", - ) + self.metadata = self.ap.adapter_qq_botpy_meta self.member_openid_mapping = OpenIDMapping( map=self.metadata.data["mapping"]["members"], diff --git a/pkg/plugin/setting.py b/pkg/plugin/setting.py index c1934937..7e715af1 100644 --- a/pkg/plugin/setting.py +++ b/pkg/plugin/setting.py @@ -16,10 +16,7 @@ def __init__(self, ap: app.Application): self.ap = ap async def initialize(self): - self.settings = await cfg_mgr.load_json_config( - 'plugins/plugins.json', - 'templates/plugin-settings.json' - ) + self.settings = self.ap.plugin_setting_meta async def sync_setting( self, diff --git a/templates/metadata/sensitive-words.json b/templates/metadata/sensitive-words.json new file mode 100644 index 00000000..86820287 --- /dev/null +++ b/templates/metadata/sensitive-words.json @@ -0,0 +1,78 @@ +{ + "说明": "mask将替换敏感词中的每一个字,若mask_word值不为空,则将敏感词整个替换为mask_word的值", + "mask": "*", + "mask_word": "", + "words": [ + "习近平", + "胡锦涛", + "江泽民", + "温家宝", + "李克强", + "李长春", + "毛泽东", + "邓小平", + "周恩来", + "马克思", + "社会主义", + "共产党", + "共产主义", + "大陆官方", + "北京政权", + "中华帝国", + "中国政府", + "共狗", + "六四事件", + "天安门", + "六四", + "政治局常委", + "两会", + "共青团", + "学潮", + "八九", + "二十大", + "民进党", + "台独", + "台湾独立", + "台湾国", + "国民党", + "台湾民国", + "中华民国", + "pornhub", + "Pornhub", + "[Yy]ou[Pp]orn", + "porn", + "Porn", + "[Xx][Vv]ideos", + "[Rr]ed[Tt]ube", + "[Xx][Hh]amster", + "[Ss]pank[Ww]ire", + "[Ss]pank[Bb]ang", + "[Tt]ube8", + "[Yy]ou[Jj]izz", + "[Bb]razzers", + "[Nn]aughty[ ]?[Aa]merica", + "作爱", + "做爱", + "性交", + "性爱", + "自慰", + "阴茎", + "淫妇", + "肛交", + "交配", + "性关系", + "性活动", + "色情", + "色图", + "涩图", + "裸体", + "小穴", + "淫荡", + "性爱", + "翻墙", + "VPN", + "科学上网", + "挂梯子", + "GFW" + ] +} \ No newline at end of file diff --git a/templates/sensitive-words.json b/templates/sensitive-words.json deleted file mode 100644 index 61d15ff9..00000000 --- a/templates/sensitive-words.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "说明": "mask将替换敏感词中的每一个字,若mask_word值不为空,则将敏感词整个替换为mask_word的值", - "mask": "*", - "mask_word": "", - "words": [ - "习近平", - "胡锦涛", - "江泽民", - "温家宝", - "李克强", - "李长春", - "毛泽东", - "邓小平", - "周恩来", - "马克思", - "社会主义", - "共产党", - "共产主义", - "大陆官方", - "北京政权", - "中华帝国", - "中国政府", - "共狗", - "六四事件", - "天安门", - "六四", - "政治局常委", - "两会", - "共青团", - "学潮", - "八九", - "二十大", - "民进党", - "台独", - "台湾独立", - "台湾国", - "国民党", - "台湾民国", - "中华民国", - "pornhub", - "Pornhub", - "[Yy]ou[Pp]orn", - "porn", - "Porn", - "[Xx][Vv]ideos", - "[Rr]ed[Tt]ube", - "[Xx][Hh]amster", - "[Ss]pank[Ww]ire", - "[Ss]pank[Bb]ang", - "[Tt]ube8", - "[Yy]ou[Jj]izz", - "[Bb]razzers", - "[Nn]aughty[ ]?[Aa]merica", - "作爱", - "做爱", - "性交", - "性爱", - "自慰", - "阴茎", - "淫妇", - "肛交", - "交配", - "性关系", - "性活动", - "色情", - "色图", - "涩图", - "裸体", - "小穴", - "淫荡", - "性爱", - "翻墙", - "VPN", - "科学上网", - "挂梯子", - "GFW" - ] -} \ No newline at end of file