Merge pull request #58 from 2jun0/GDET-70

GDET-70: 별칭 가져오기

.github/workflows/test-lambda.yaml

@@ -47,3 +47,6 @@ jobs:
       - name: test app
         run: poetry run python -m pytest tests -c pytest.ini
         working-directory: ${{env.working-directory}}
+        env:
+          IGDB_CLIENT_ID: ${{ secrets.IGDB_CLIENT_ID }}
+          IGDB_CLIENT_SECRET: ${{ secrets.IGDB_CLIENT_SECRET }}

aws_lambdas/database_lambda/alias/model.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from sqlalchemy import ForeignKey, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from ..model import Base, CreatedAtMixin, UpdatedAtMixin
+
+
+class GameAlias(CreatedAtMixin, UpdatedAtMixin, Base):
+    __tablename__ = "game_alias"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(64))
+
+    game_id: Mapped[int] = mapped_column(ForeignKey("game.id"))
+
+    def __repr__(self) -> str:
+        return f"GameAlias(id={self.id}, name={self.name}, game_id={self.game_id}"
+
+    def to_dto(self) -> "GameAliasDto":
+        return GameAliasDto(
+            id=self.id,
+            name=self.name,
+            game_id=self.game_id,
+            created_at=self.created_at,
+            updated_at=self.updated_at,
+        )
+
+
+class GameAliasDto(BaseModel):
+    id: int
+    name: str
+    game_id: int
+    created_at: datetime
+    updated_at: datetime

aws_lambdas/database_lambda/game/es.py

@@ -4,18 +4,20 @@
 from elasticsearch import Elasticsearch, helpers
 
 from ..es import GAME_INDEX
+from ..utils import is_aldecimal
 from .model import Game
 
 
 def _bulk_game_data(games: Iterable[Game]) -> Generator[dict[str, Any], Any, None]:
     for game in games:
-        q_name = "".join([c if c.isalnum() else " " for c in game.name])
+        q_name = "".join([c if is_aldecimal(c) else " " for c in game.name])
         yield {
             "_index": GAME_INDEX,
             "_id": game.id,
             "id": game.id,
             "name": game.name,
             "q_name": q_name,
+            "aliases": [alias.name for alias in game.aliases],
         }
 
 

aws_lambdas/database_lambda/game/model.py

@@ -1,10 +1,11 @@
 from datetime import datetime
-from typing import Optional
 
 from pydantic import BaseModel
 from sqlalchemy import Column, ForeignKey, Integer, String, Table
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
+from ..alias.model import GameAlias
+
 from ..genre.model import Genre
 from ..model import Base, CreatedAtMixin, UpdatedAtMixin
 
@@ -23,24 +24,21 @@ class Game(CreatedAtMixin, UpdatedAtMixin, Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     steam_id: Mapped[int] = mapped_column(unique=True)
     name: Mapped[str] = mapped_column(String(64))
-    kr_name: Mapped[Optional[str]] = mapped_column(String(64))
     released_at: Mapped[datetime] = mapped_column()
     genres: Mapped[list[Genre]] = relationship(secondary=game_genre_link)
+    aliases: Mapped[list[GameAlias]] = relationship(cascade="all, delete-orphan")
 
     def __repr__(self) -> str:
-        return (
-            f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, kr_name={self.kr_name},"
-            f" released_at={self.released_at})"
-        )
+        return f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, released_at={self.released_at})"
 
     def to_dto(self) -> "GameDto":
         return GameDto(
             id=self.id,
             steam_id=self.steam_id,
             name=self.name,
-            kr_name=self.kr_name,
             released_at=self.released_at,
             genres=[g.name for g in self.genres],
+            aliases=[a.name for a in self.aliases],
             created_at=self.created_at,
             updated_at=self.updated_at,
         )
@@ -50,8 +48,8 @@ class GameDto(BaseModel):
     id: int
     steam_id: int
     name: str
-    kr_name: Optional[str]
     released_at: datetime
     genres: list[str]
+    aliases: list[str]
     created_at: datetime
     updated_at: datetime

aws_lambdas/database_lambda/game/model_factory.py

@@ -3,7 +3,9 @@
 
 from sqlalchemy.orm import Session
 
+from ..alias.model import GameAlias
 from ..genre.model_factory import to_models as to_genre_models
+from ..utils import is_aldecimal
 from . import repository
 from .model import Game
 from .schema import STEAM_ID, SaveGame
@@ -13,9 +15,9 @@ def _create_model(session: Session, game: SaveGame) -> Game:
     return Game(
         steam_id=game["steam_id"],
         name=game["name"],
-        kr_name=game["kr_name"],
         released_at=datetime.fromtimestamp(game["released_at"]),
         genres=to_genre_models(session, game["genres"]),
+        aliases=[],
     )
 
 
@@ -33,13 +35,36 @@ def _attach_models(session: Session, models: dict[STEAM_ID, Game]):
     for game in saved:
         query = models[game.steam_id]
         game.name = query.name
-        game.kr_name = query.kr_name
         game.genres = query.genres
         models[game.steam_id] = game
 
 
+def _update_aliases(session: Session, game: Game, aliases: set[str]):
+    # remove aliases
+    existed_aliases: list[str] = []
+    for game_alias in game.aliases:
+        if game_alias.name not in aliases:
+            session.delete(game_alias)
+        else:
+            existed_aliases.append(game_alias.name)
+
+    # add aliases
+    for alias_name in aliases:
+        if alias_name not in existed_aliases:
+            game.aliases.append(GameAlias(name=alias_name))
+
+
+def _allow_alias_name(alias_name: str):
+    return all(c == " " or is_aldecimal(c) for c in alias_name)
+
+
 def to_models(session: Session, games: Iterable[SaveGame]) -> list[Game]:
     models = _create_models(session, games)
     _attach_models(session, models)
 
+    for game in games:
+        model = models[game["steam_id"]]
+        aliases = set(alias_name.lower() for alias_name in game["aliases"] if _allow_alias_name(alias_name))
+        _update_aliases(session, model, aliases)
+
     return list(models.values())

aws_lambdas/database_lambda/game/schema.py

@@ -1,12 +1,12 @@
 from collections.abc import Sequence
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 STEAM_ID = int
 
 
 class SaveGame(TypedDict):
     steam_id: int
     name: str
-    kr_name: Optional[str]
+    aliases: Sequence[str]
     released_at: float
     genres: Sequence[str]

aws_lambdas/database_lambda/utils.py

@@ -0,0 +1,2 @@
+def is_aldecimal(s: str) -> bool:
+    return all(c.isdecimal() or c.isalpha() for c in s)

aws_lambdas/game_updater/aws_lambda/model.py

@@ -1,23 +1,11 @@
-from datetime import datetime
-from typing import Optional, Sequence
+from typing import Sequence
 
 from pydantic import BaseModel
 
 
-class Game(BaseModel):
-    id: int
-    steam_id: int
-    name: str
-    kr_name: Optional[str]
-    released_at: datetime
-    genres: Sequence[str]
-    updated_at: datetime
-    created_at: datetime
-
-
 class SaveGame(BaseModel):
     steam_id: int
     name: str
-    kr_name: Optional[str]
     released_at: float
     genres: Sequence[str]
+    aliases: Sequence[str]

aws_lambdas/game_updater/config.py

@@ -2,11 +2,15 @@
 
 
 class Config(BaseSettings):
+    DATABASE_LAMBDA_NAME: str = "database"
+
     WORKER_CNT: int = 10
     MIN_REVENUE: int = 10000000  # 10M
-    DATABASE_LAMBDA_NAME: str = "database"
 
-    model_config = SettingsConfigDict(env_file=".scraper.env", env_file_encoding="utf-8")
+    IGDB_CLIENT_ID: str
+    IGDB_CLIENT_SECRET: str
+
+    model_config = SettingsConfigDict(env_file=".game_updater.env", env_file_encoding="utf-8")
 
 
 setting = Config()  # type: ignore

aws_lambdas/game_updater/igdb/igdb_api.py

@@ -0,0 +1,121 @@
+from collections.abc import Iterable, Sequence
+from functools import cache, wraps
+
+from .model import IGDBAlternativeName, IGDBExternalGames, IGDBGame
+from ..config import setting
+
+import requests
+
+STEAM_CATEGORY = 1
+MAX_LIMIT = 500
+
+
+def batch(size: int):
+    def batch_decorator(func):
+
+        @wraps(func)
+        def wrapper(_input: Sequence[int], **kwargs):
+            outputs = []
+
+            for srt_idx in range(0, len(_input), size):
+                batch_input = _input[srt_idx : srt_idx + size]
+                batch_output = func(batch_input, **kwargs)
+                outputs.extend(batch_output)
+
+            return outputs
+
+        return wrapper
+
+    return batch_decorator
+
+
+@cache
+def _get_token():
+    res = requests.post(
+        "https://id.twitch.tv/oauth2/token"
+        f"?client_id={setting.IGDB_CLIENT_ID}&client_secret={setting.IGDB_CLIENT_SECRET}&grant_type=client_credentials",
+    )
+    token = res.json()["access_token"]
+    return token
+
+
+def get_external_games(steam_ids: Iterable, category: int, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]:
+    # category,checksum,countries,created_at,game,media,name,platform,uid,updated_at,url,year;
+
+    uids = ",".join([f'"{id}"' for id in steam_ids])
+    token = _get_token()
+
+    response = requests.post(
+        "https://api.igdb.com/v4/external_games",
+        **{
+            "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+            "data": f"fields game,uid; where uid=({uids}) & category={STEAM_CATEGORY}; limit {limit};",
+        },
+    )
+    response.raise_for_status()
+
+    return response.json()
+
+
+def get_steam_games(steam_ids: Iterable, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]:
+    return get_external_games(steam_ids, STEAM_CATEGORY, limit)
+
+
+def get_games(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBGame]:
+    # checksum,cover,created_at,game,name,region,updated_at;
+
+    ids_ = ",".join(map(str, ids))
+    token = _get_token()
+
+    response = requests.post(
+        "https://api.igdb.com/v4/games",
+        **{
+            "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+            "data": f"fields alternative_names; where id=({ids_}); limit {limit};",
+        },
+    )
+    response.raise_for_status()
+
+    return response.json()
+
+
+def get_alternative_names(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBAlternativeName]:
+    # checksum,cover,created_at,game,name,region,updated_at;
+
+    ids_ = ",".join(map(str, ids))
+    token = _get_token()
+
+    response = requests.post(
+        "https://api.igdb.com/v4/alternative_names",
+        **{
+            "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+            "data": f"fields name,game; where id=({ids_}); limit {limit};",
+        },
+    )
+    response.raise_for_status()
+
+    return response.json()
+
+
+@batch(MAX_LIMIT)
+def get_steam_games_batch(steam_ids: Sequence[int]) -> list[IGDBExternalGames]:
+    if len(steam_ids) == 0:
+        return []
+
+    return get_steam_games(steam_ids=steam_ids)
+
+
+@batch(MAX_LIMIT)
+def get_games_batch(game_ids: Sequence[int]) -> list[IGDBGame]:
+    if len(game_ids) == 0:
+        return []
+
+    return get_games(ids=game_ids)
+
+
+@batch(MAX_LIMIT)
+def get_alternative_names_batch(alternative_name_ids: Sequence[int]) -> list[IGDBAlternativeName]:
+    if len(alternative_name_ids) == 0:
+        return []
+
+    return get_alternative_names(ids=alternative_name_ids)

aws_lambdas/game_updater/igdb/model.py

@@ -0,0 +1,18 @@
+from typing import TypedDict
+
+
+class IGDBExternalGames(TypedDict):
+    id: int
+    uid: str
+    game: int
+
+
+class IGDBGame(TypedDict):
+    id: int
+    alternative_names: list[int]
+
+
+class IGDBAlternativeName(TypedDict):
+    id: int
+    name: str
+    game: int

aws_lambdas/game_updater/lambda_func.py

@@ -5,13 +5,13 @@ def lambda_handler(event: Any, context: Any):
 
     from game_updater.aws_lambda.lambda_api import LambdaAPI
     from game_updater.logger import logger
-    from game_updater.scraper.service import scrap_games
+    from game_updater.scraper.service import update_games
     from game_updater.steam.steam_api import SteamAPI
 
     def scrap_games_job(lambda_api: LambdaAPI):
         logger.info("-- scrap game job start -- ")
 
-        scrap_games(SteamAPI(), lambda_api)
+        update_games(SteamAPI(), lambda_api)
 
         logger.info("-- scrap game job end -- ")