feat: save raw data

res_func/avatar.py

@@ -9,18 +9,17 @@
 
 from func.data import all_avatars_map, all_avatars_name, read_avatars, dump_avatars
 from models.avatar_config import AvatarConfig, AvatarIcon
+from .base_data import get_base_data
 from .client import client
 from .url import avatar_config, text_map, base_station_url, avatar_url
 
 
 async def fetch_text_map() -> Dict[str, str]:
-    res = await client.get(text_map)
-    return res.json()
+    return await get_base_data(text_map)
 
 
 async def fetch_config(text_map_data: Dict[str, str]) -> List[AvatarConfig]:
-    res = await client.get(avatar_config)
-    data = res.json()
+    data = await get_base_data(avatar_config)
     datas = []
     for i in data.values():
         a = AvatarConfig(**i)

res_func/base_data.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import aiofiles
+import ujson
+
+from .client import client
+from .url import base_data_url
+
+data_path = Path("data")
+raw_path = data_path / "raw"
+
+
+def get_real_path(url: str) -> Path:
+    file_url = url.replace(base_data_url, "")
+    file_path = Path(file_url)
+    (raw_path / file_path.parent).mkdir(parents=True, exist_ok=True)
+    return raw_path / file_path
+
+
+async def get_base_data_from_file(file_path: Path):
+    async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
+        return ujson.loads(await f.read())
+
+
+async def get_base_data_from_url(url: str, file_path: Path) -> dict:
+    res = await client.get(url)
+    data = res.json()
+    async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
+        await f.write(ujson.dumps(data, indent=4, ensure_ascii=False))
+    return data
+
+
+async def get_base_data(url: str) -> dict:
+    file_path = get_real_path(url)
+    if file_path.exists():
+        return await get_base_data_from_file(file_path)
+    return await get_base_data_from_url(url, file_path)

res_func/relic.py

@@ -6,6 +6,7 @@
 
 from models.enums import RelicAffix, RelicPosition
 from models.relic_affix import RelicAffixAll, SingleRelicAffix
+from res_func.base_data import get_base_data
 from res_func.client import client
 from res_func.url import relic_config, relic_main_affix_config, relic_sub_affix_config
 
@@ -15,8 +16,7 @@
 
 async def fetch_all_relic():
     print("开始获取遗器配置")
-    relic_data_req = await client.get(relic_config)
-    relic_data = relic_data_req.json()
+    relic_data = await get_base_data(relic_config)
     for key, value in relic_data.items():
         relic_affix_all = RelicAffixAll(
             id=int(key),
@@ -36,8 +36,7 @@ async def fetch_all_relic():
 
 async def fetch_main_affix():
     print("开始获取遗器主词条配置")
-    main_affix_req = await client.get(relic_main_affix_config)
-    main_affix_data = main_affix_req.json()
+    main_affix_data = await get_base_data(relic_main_affix_config)
     main_affix_groups_map: Dict[str, Dict[str, SingleRelicAffix]] = {}
     for key, value in main_affix_data.items():
         data: Dict[str, SingleRelicAffix] = {}
@@ -57,8 +56,7 @@ async def fetch_main_affix():
 
 async def fetch_sub_affix():
     print("开始获取遗器副词条配置")
-    sub_affix_req = await client.get(relic_sub_affix_config)
-    sub_affix_data = sub_affix_req.json()
+    sub_affix_data = await get_base_data(relic_sub_affix_config)
     sub_affix_groups_map: Dict[str, Dict[str, SingleRelicAffix]] = {}
     for key, value in sub_affix_data.items():
         data: Dict[str, SingleRelicAffix] = {}

res_func/url.py

@@ -8,7 +8,6 @@
 relic_config = f"{base_data_url}ExcelOutput/RelicConfig.json"
 relic_main_affix_config = f"{base_data_url}ExcelOutput/RelicMainAffixConfig.json"
 relic_sub_affix_config = f"{base_data_url}ExcelOutput/RelicSubAffixConfig.json"
-relic_set_config = f"{base_data_url}ExcelOutput/RelicSetConfig.json"
 base_station_url = "https://starrailstation.com"
 avatar_url = f"{base_station_url}/cn/characters"
 light_cone_url = f"{base_station_url}/cn/equipment"