Merge pull request #644 from RockChinQ/feat/online-data-analysis

Feat: v2 数据统计接口

.gitignore

@@ -30,4 +30,5 @@ qcapi
 claude.json
 bard.json
 /*yaml
-!/docker-compose.yaml
+!/docker-compose.yaml
+res/instance_id.json

README.md

@@ -29,7 +29,6 @@
 <img alt="Static Badge" src="https://img.shields.io/badge/Linux%E9%83%A8%E7%BD%B2%E8%A7%86%E9%A2%91-208647">
 </a>
 
-
 <a href="https://qchatgpt.rockchin.top">项目主页</a> ｜
 <a href="https://qchatgpt.rockchin.top/posts/feature.html">功能介绍</a> ｜
 <a href="https://qchatgpt.rockchin.top/posts/deploy/">部署文档</a> ｜
@@ -38,6 +37,4 @@
 <a href="https://github.com/RockChinQ/QChatGPT/issues/new?assignees=&labels=%E7%8B%AC%E7%AB%8B%E6%8F%92%E4%BB%B6&projects=&template=submit-plugin.yml&title=%5BPlugin%5D%3A+%E8%AF%B7%E6%B1%82%E7%99%BB%E8%AE%B0%E6%96%B0%E6%8F%92%E4%BB%B6">提交插件</a>
 
 <img alt="回复效果（带有联网插件）" src="https://qchatgpt.rockchin.top/assets/image/QChatGPT-1211.png" width="500px"/>
-
-
 </div>

main.py

@@ -122,7 +122,7 @@ def complete_tips():
     non_exist_keys = []
 
     is_integrity = True
-    logging.info("检查tips模块完整性.")
+    logging.debug("检查tips模块完整性.")
     tips_template = importlib.import_module('tips-custom-template')
     tips = importlib.import_module('tips')
     for key in dir(tips_template):
@@ -145,6 +145,10 @@ async def start_process(first_time_init=False):
     global known_exception_caught
     import pkg.utils.context
 
+    # 计算host和instance标识符
+    import pkg.audit.identifier
+    pkg.audit.identifier.init()
+
     # 加载配置
     cfg_inst: pymodule_cfg.PythonModuleConfigFile = pymodule_cfg.PythonModuleConfigFile(
         'config.py',
@@ -158,6 +162,7 @@ async def start_process(first_time_init=False):
     complete_tips()
 
     cfg = pkg.utils.context.get_config_manager().data
+
     # 更新openai库到最新版本
     if 'upgrade_dependencies' not in cfg or cfg['upgrade_dependencies']:
         print("正在更新依赖库，请等待...")
@@ -204,6 +209,24 @@ async def start_process(first_time_init=False):
                         break
                     except ValueError:
                         print("请输入数字")
+
+            # 初始化中央服务器 API 交互实例
+            from pkg.utils.center import apigroup
+            from pkg.utils.center import v2 as center_v2
+
+            center_v2_api = center_v2.V2CenterAPI(
+                basic_info={
+                    "host_id": pkg.audit.identifier.identifier['host_id'],
+                    "instance_id": pkg.audit.identifier.identifier['instance_id'],
+                    "semantic_version": pkg.utils.updater.get_current_tag(),
+                    "platform": sys.platform,
+                },
+                runtime_info={
+                    "admin_id": "{}".format(cfg['admin_qq']),
+                    "msg_source": cfg['msg_source_adapter'],
+                }
+            )
+            pkg.utils.context.set_center_v2_api(center_v2_api)
 
             import pkg.openai.manager
             import pkg.database.manager
@@ -375,6 +398,12 @@ def run_bot_wrapper():
         if len(new_announcement) > 0:
             for announcement in new_announcement:
                 logging.critical("[公告]<{}> {}".format(announcement['time'], announcement['content']))
+
+            # 发送统计数据
+            pkg.utils.context.get_center_v2_api().main.post_announcement_showed(
+                [announcement['id'] for announcement in new_announcement]
+            )
+
     except Exception as e:
         logging.warning("获取公告失败:{}".format(e))
 

pkg/audit/identifier.py

@@ -0,0 +1,83 @@
+import os
+import uuid
+import json
+import time
+
+
+identifier = {
+    'host_id': '',
+    'instance_id': '',
+    'host_create_ts': 0,
+    'instance_create_ts': 0,
+}
+
+HOST_ID_FILE = os.path.expanduser('~/.qchatgpt/host_id.json')
+INSTANCE_ID_FILE = 'res/instance_id.json'
+
+def init():
+    global identifier
+
+    if not os.path.exists(os.path.expanduser('~/.qchatgpt')):
+        os.mkdir(os.path.expanduser('~/.qchatgpt'))
+
+    if not os.path.exists(HOST_ID_FILE):
+        new_host_id = 'host_'+str(uuid.uuid4())
+        new_host_create_ts = int(time.time())
+
+        with open(HOST_ID_FILE, 'w') as f:
+            json.dump({
+                'host_id': new_host_id,
+                'host_create_ts': new_host_create_ts
+            }, f)
+
+        identifier['host_id'] = new_host_id
+        identifier['host_create_ts'] = new_host_create_ts
+    else:
+        loaded_host_id = ''
+        loaded_host_create_ts = 0
+
+        with open(HOST_ID_FILE, 'r') as f:
+            file_content = json.load(f)
+            loaded_host_id = file_content['host_id']
+            loaded_host_create_ts = file_content['host_create_ts']
+
+        identifier['host_id'] = loaded_host_id
+        identifier['host_create_ts'] = loaded_host_create_ts
+
+    # 检查实例 id
+    if os.path.exists(INSTANCE_ID_FILE):
+        instance_id = {}
+        with open(INSTANCE_ID_FILE, 'r') as f:
+            instance_id = json.load(f)
+
+        if instance_id['host_id'] != identifier['host_id']: # 如果实例 id 不是当前主机的，删除
+            os.remove(INSTANCE_ID_FILE)
+
+    if not os.path.exists(INSTANCE_ID_FILE):
+        new_instance_id = 'instance_'+str(uuid.uuid4())
+        new_instance_create_ts = int(time.time())
+
+        with open(INSTANCE_ID_FILE, 'w') as f:
+            json.dump({
+                'host_id': identifier['host_id'],
+                'instance_id': new_instance_id,
+                'instance_create_ts': new_instance_create_ts
+            }, f)
+
+        identifier['instance_id'] = new_instance_id
+        identifier['instance_create_ts'] = new_instance_create_ts
+    else:
+        loaded_instance_id = ''
+        loaded_instance_create_ts = 0
+
+        with open(INSTANCE_ID_FILE, 'r') as f:
+            file_content = json.load(f)
+            loaded_instance_id = file_content['instance_id']
+            loaded_instance_create_ts = file_content['instance_create_ts']
+
+        identifier['instance_id'] = loaded_instance_id
+        identifier['instance_create_ts'] = loaded_instance_create_ts
+
+def print_out():
+    global identifier
+    print(identifier)

pkg/database/manager.py

@@ -91,7 +91,7 @@ def initialize_database(self):
             `json` text not null
         )
         """)
-        print('Database initialized.')
+        # print('Database initialized.')
 
     # session持久化
     def persistence_session(self, subject_type: str, subject_number: int, create_timestamp: int,

pkg/openai/api/chat_completion.py

@@ -6,6 +6,8 @@
 
 from .model import RequestBase
 from .. import funcmgr
+from ...plugin import host
+from ...utils import context
 
 
 class ChatCompletionRequest(RequestBase):
@@ -189,6 +191,16 @@ def __next__(self) -> dict:
                     ret = "error: execute function failed: {}".format(str(e))
                     logging.error("函数执行失败: {}".format(str(e)))
 
+                # 上报数据
+                plugin_info = host.get_plugin_info_for_audit(func_name.split('-')[0])
+                audit_func_name = func_name.split('-')[1]
+                audit_func_desc = funcmgr.get_func_schema(func_name)['description']
+                context.get_center_v2_api().usage.post_function_record(
+                    plugin=plugin_info,
+                    function_name=audit_func_name,
+                    function_description=audit_func_desc,
+                )
+
                 self.append_message(
                     role="function",
                     content=json.dumps(ret, ensure_ascii=False),

pkg/openai/keymgr.py

@@ -75,7 +75,7 @@ def auto_switch(self) -> tuple[bool, str]:
             if self.api_key[key_name] not in self.exceeded:
                 self.using_key = self.api_key[key_name]
 
-                logging.info("使用api-key:" + key_name)
+                logging.debug("使用api-key:" + key_name)
 
                 # 触发插件事件
                 args = {

pkg/openai/session.py

@@ -261,6 +261,8 @@ def query(self, text: str=None) -> tuple[str, str, list[str]]:
 
         pending_res_text = ""
 
+        start_time = time.time()
+
         # TODO 对不起，我知道这样非常非常屎山，但我之后会重构的
         for resp in context.get_openai_manager().request_completion(prompts):
 
@@ -349,6 +351,26 @@ def query(self, text: str=None) -> tuple[str, str, list[str]]:
             self.just_switched_to_exist_session = False
             self.set_ongoing()
 
+        # 上报使用量数据
+        session_type = session_name_spt[0]
+        session_id = session_name_spt[1]
+
+        ability_provider = "QChatGPT.Text"
+        usage = total_tokens
+        model_name = context.get_config_manager().data['completion_api_params']['model']
+        response_seconds = int(time.time() - start_time)
+        retry_times = -1 # 暂不记录
+
+        context.get_center_v2_api().usage.post_query_record(
+            session_type=session_type,
+            session_id=session_id,
+            query_ability_provider=ability_provider,
+            usage=usage,
+            model_name=model_name,
+            response_seconds=response_seconds,
+            retry_times=retry_times
+        )
+
         return res_ans if res_ans[0] != '\n' else res_ans[1:], finish_reason, funcs
 
     # 删除上一回合并返回上一回合的问题