diff --git a/.github/workflows/test-lambda.yaml b/.github/workflows/test-lambda.yaml index 5cba7fe..e62a802 100644 --- a/.github/workflows/test-lambda.yaml +++ b/.github/workflows/test-lambda.yaml @@ -47,3 +47,6 @@ jobs: - name: test app run: poetry run python -m pytest tests -c pytest.ini working-directory: ${{env.working-directory}} + env: + IGDB_CLIENT_ID: ${{ secrets.IGDB_CLIENT_ID }} + IGDB_CLIENT_SECRET: ${{ secrets.IGDB_CLIENT_SECRET }} diff --git a/aws_lambdas/database_lambda/alias/model.py b/aws_lambdas/database_lambda/alias/model.py new file mode 100644 index 0000000..7cb7efc --- /dev/null +++ b/aws_lambdas/database_lambda/alias/model.py @@ -0,0 +1,36 @@ +from datetime import datetime + +from pydantic import BaseModel +from sqlalchemy import ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column + +from ..model import Base, CreatedAtMixin, UpdatedAtMixin + + +class GameAlias(CreatedAtMixin, UpdatedAtMixin, Base): + __tablename__ = "game_alias" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(64)) + + game_id: Mapped[int] = mapped_column(ForeignKey("game.id")) + + def __repr__(self) -> str: + return f"GameAlias(id={self.id}, name={self.name}, game_id={self.game_id}" + + def to_dto(self) -> "GameAliasDto": + return GameAliasDto( + id=self.id, + name=self.name, + game_id=self.game_id, + created_at=self.created_at, + updated_at=self.updated_at, + ) + + +class GameAliasDto(BaseModel): + id: int + name: str + game_id: int + created_at: datetime + updated_at: datetime diff --git a/aws_lambdas/database_lambda/game/es.py b/aws_lambdas/database_lambda/game/es.py index 31ad5bd..34feceb 100644 --- a/aws_lambdas/database_lambda/game/es.py +++ b/aws_lambdas/database_lambda/game/es.py @@ -4,18 +4,20 @@ from elasticsearch import Elasticsearch, helpers from ..es import GAME_INDEX +from ..utils import is_aldecimal from .model import Game def _bulk_game_data(games: Iterable[Game]) -> Generator[dict[str, Any], Any, None]: for game in games: - q_name = "".join([c if c.isalnum() else " " for c in game.name]) + q_name = "".join([c if is_aldecimal(c) else " " for c in game.name]) yield { "_index": GAME_INDEX, "_id": game.id, "id": game.id, "name": game.name, "q_name": q_name, + "aliases": [alias.name for alias in game.aliases], } diff --git a/aws_lambdas/database_lambda/game/model.py b/aws_lambdas/database_lambda/game/model.py index 95776f3..472b11a 100644 --- a/aws_lambdas/database_lambda/game/model.py +++ b/aws_lambdas/database_lambda/game/model.py @@ -1,10 +1,11 @@ from datetime import datetime -from typing import Optional from pydantic import BaseModel from sqlalchemy import Column, ForeignKey, Integer, String, Table from sqlalchemy.orm import Mapped, mapped_column, relationship +from ..alias.model import GameAlias + from ..genre.model import Genre from ..model import Base, CreatedAtMixin, UpdatedAtMixin @@ -23,24 +24,21 @@ class Game(CreatedAtMixin, UpdatedAtMixin, Base): id: Mapped[int] = mapped_column(primary_key=True) steam_id: Mapped[int] = mapped_column(unique=True) name: Mapped[str] = mapped_column(String(64)) - kr_name: Mapped[Optional[str]] = mapped_column(String(64)) released_at: Mapped[datetime] = mapped_column() genres: Mapped[list[Genre]] = relationship(secondary=game_genre_link) + aliases: Mapped[list[GameAlias]] = relationship(cascade="all, delete-orphan") def __repr__(self) -> str: - return ( - f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, kr_name={self.kr_name}," - f" released_at={self.released_at})" - ) + return f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, released_at={self.released_at})" def to_dto(self) -> "GameDto": return GameDto( id=self.id, steam_id=self.steam_id, name=self.name, - kr_name=self.kr_name, released_at=self.released_at, genres=[g.name for g in self.genres], + aliases=[a.name for a in self.aliases], created_at=self.created_at, updated_at=self.updated_at, ) @@ -50,8 +48,8 @@ class GameDto(BaseModel): id: int steam_id: int name: str - kr_name: Optional[str] released_at: datetime genres: list[str] + aliases: list[str] created_at: datetime updated_at: datetime diff --git a/aws_lambdas/database_lambda/game/model_factory.py b/aws_lambdas/database_lambda/game/model_factory.py index 929c1c0..18216e9 100644 --- a/aws_lambdas/database_lambda/game/model_factory.py +++ b/aws_lambdas/database_lambda/game/model_factory.py @@ -3,7 +3,9 @@ from sqlalchemy.orm import Session +from ..alias.model import GameAlias from ..genre.model_factory import to_models as to_genre_models +from ..utils import is_aldecimal from . import repository from .model import Game from .schema import STEAM_ID, SaveGame @@ -13,9 +15,9 @@ def _create_model(session: Session, game: SaveGame) -> Game: return Game( steam_id=game["steam_id"], name=game["name"], - kr_name=game["kr_name"], released_at=datetime.fromtimestamp(game["released_at"]), genres=to_genre_models(session, game["genres"]), + aliases=[], ) @@ -33,13 +35,36 @@ def _attach_models(session: Session, models: dict[STEAM_ID, Game]): for game in saved: query = models[game.steam_id] game.name = query.name - game.kr_name = query.kr_name game.genres = query.genres models[game.steam_id] = game +def _update_aliases(session: Session, game: Game, aliases: set[str]): + # remove aliases + existed_aliases: list[str] = [] + for game_alias in game.aliases: + if game_alias.name not in aliases: + session.delete(game_alias) + else: + existed_aliases.append(game_alias.name) + + # add aliases + for alias_name in aliases: + if alias_name not in existed_aliases: + game.aliases.append(GameAlias(name=alias_name)) + + +def _allow_alias_name(alias_name: str): + return all(c == " " or is_aldecimal(c) for c in alias_name) + + def to_models(session: Session, games: Iterable[SaveGame]) -> list[Game]: models = _create_models(session, games) _attach_models(session, models) + for game in games: + model = models[game["steam_id"]] + aliases = set(alias_name.lower() for alias_name in game["aliases"] if _allow_alias_name(alias_name)) + _update_aliases(session, model, aliases) + return list(models.values()) diff --git a/aws_lambdas/database_lambda/game/schema.py b/aws_lambdas/database_lambda/game/schema.py index 9a10609..7eae637 100644 --- a/aws_lambdas/database_lambda/game/schema.py +++ b/aws_lambdas/database_lambda/game/schema.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import Optional, TypedDict +from typing import TypedDict STEAM_ID = int @@ -7,6 +7,6 @@ class SaveGame(TypedDict): steam_id: int name: str - kr_name: Optional[str] + aliases: Sequence[str] released_at: float genres: Sequence[str] diff --git a/aws_lambdas/database_lambda/utils.py b/aws_lambdas/database_lambda/utils.py new file mode 100644 index 0000000..b70ed2d --- /dev/null +++ b/aws_lambdas/database_lambda/utils.py @@ -0,0 +1,2 @@ +def is_aldecimal(s: str) -> bool: + return all(c.isdecimal() or c.isalpha() for c in s) diff --git a/aws_lambdas/game_updater/aws_lambda/model.py b/aws_lambdas/game_updater/aws_lambda/model.py index 8857f70..a698526 100644 --- a/aws_lambdas/game_updater/aws_lambda/model.py +++ b/aws_lambdas/game_updater/aws_lambda/model.py @@ -1,23 +1,11 @@ -from datetime import datetime -from typing import Optional, Sequence +from typing import Sequence from pydantic import BaseModel -class Game(BaseModel): - id: int - steam_id: int - name: str - kr_name: Optional[str] - released_at: datetime - genres: Sequence[str] - updated_at: datetime - created_at: datetime - - class SaveGame(BaseModel): steam_id: int name: str - kr_name: Optional[str] released_at: float genres: Sequence[str] + aliases: Sequence[str] diff --git a/aws_lambdas/game_updater/config.py b/aws_lambdas/game_updater/config.py index e4a09aa..3d61506 100644 --- a/aws_lambdas/game_updater/config.py +++ b/aws_lambdas/game_updater/config.py @@ -2,11 +2,15 @@ class Config(BaseSettings): + DATABASE_LAMBDA_NAME: str = "database" + WORKER_CNT: int = 10 MIN_REVENUE: int = 10000000 # 10M - DATABASE_LAMBDA_NAME: str = "database" - model_config = SettingsConfigDict(env_file=".scraper.env", env_file_encoding="utf-8") + IGDB_CLIENT_ID: str + IGDB_CLIENT_SECRET: str + + model_config = SettingsConfigDict(env_file=".game_updater.env", env_file_encoding="utf-8") setting = Config() # type: ignore diff --git a/aws_lambdas/game_updater/igdb/igdb_api.py b/aws_lambdas/game_updater/igdb/igdb_api.py new file mode 100644 index 0000000..a65ca40 --- /dev/null +++ b/aws_lambdas/game_updater/igdb/igdb_api.py @@ -0,0 +1,121 @@ +from collections.abc import Iterable, Sequence +from functools import cache, wraps + +from .model import IGDBAlternativeName, IGDBExternalGames, IGDBGame +from ..config import setting + +import requests + +STEAM_CATEGORY = 1 +MAX_LIMIT = 500 + + +def batch(size: int): + def batch_decorator(func): + + @wraps(func) + def wrapper(_input: Sequence[int], **kwargs): + outputs = [] + + for srt_idx in range(0, len(_input), size): + batch_input = _input[srt_idx : srt_idx + size] + batch_output = func(batch_input, **kwargs) + outputs.extend(batch_output) + + return outputs + + return wrapper + + return batch_decorator + + +@cache +def _get_token(): + res = requests.post( + "https://id.twitch.tv/oauth2/token" + f"?client_id={setting.IGDB_CLIENT_ID}&client_secret={setting.IGDB_CLIENT_SECRET}&grant_type=client_credentials", + ) + token = res.json()["access_token"] + return token + + +def get_external_games(steam_ids: Iterable, category: int, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]: + # category,checksum,countries,created_at,game,media,name,platform,uid,updated_at,url,year; + + uids = ",".join([f'"{id}"' for id in steam_ids]) + token = _get_token() + + response = requests.post( + "https://api.igdb.com/v4/external_games", + **{ + "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"}, + "data": f"fields game,uid; where uid=({uids}) & category={STEAM_CATEGORY}; limit {limit};", + }, + ) + response.raise_for_status() + + return response.json() + + +def get_steam_games(steam_ids: Iterable, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]: + return get_external_games(steam_ids, STEAM_CATEGORY, limit) + + +def get_games(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBGame]: + # checksum,cover,created_at,game,name,region,updated_at; + + ids_ = ",".join(map(str, ids)) + token = _get_token() + + response = requests.post( + "https://api.igdb.com/v4/games", + **{ + "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"}, + "data": f"fields alternative_names; where id=({ids_}); limit {limit};", + }, + ) + response.raise_for_status() + + return response.json() + + +def get_alternative_names(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBAlternativeName]: + # checksum,cover,created_at,game,name,region,updated_at; + + ids_ = ",".join(map(str, ids)) + token = _get_token() + + response = requests.post( + "https://api.igdb.com/v4/alternative_names", + **{ + "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"}, + "data": f"fields name,game; where id=({ids_}); limit {limit};", + }, + ) + response.raise_for_status() + + return response.json() + + +@batch(MAX_LIMIT) +def get_steam_games_batch(steam_ids: Sequence[int]) -> list[IGDBExternalGames]: + if len(steam_ids) == 0: + return [] + + return get_steam_games(steam_ids=steam_ids) + + +@batch(MAX_LIMIT) +def get_games_batch(game_ids: Sequence[int]) -> list[IGDBGame]: + if len(game_ids) == 0: + return [] + + return get_games(ids=game_ids) + + +@batch(MAX_LIMIT) +def get_alternative_names_batch(alternative_name_ids: Sequence[int]) -> list[IGDBAlternativeName]: + if len(alternative_name_ids) == 0: + return [] + + return get_alternative_names(ids=alternative_name_ids) diff --git a/aws_lambdas/game_updater/igdb/model.py b/aws_lambdas/game_updater/igdb/model.py new file mode 100644 index 0000000..6ede39e --- /dev/null +++ b/aws_lambdas/game_updater/igdb/model.py @@ -0,0 +1,18 @@ +from typing import TypedDict + + +class IGDBExternalGames(TypedDict): + id: int + uid: str + game: int + + +class IGDBGame(TypedDict): + id: int + alternative_names: list[int] + + +class IGDBAlternativeName(TypedDict): + id: int + name: str + game: int diff --git a/aws_lambdas/game_updater/lambda_func.py b/aws_lambdas/game_updater/lambda_func.py index 20ddfe3..2c7ecf7 100644 --- a/aws_lambdas/game_updater/lambda_func.py +++ b/aws_lambdas/game_updater/lambda_func.py @@ -5,13 +5,13 @@ def lambda_handler(event: Any, context: Any): from game_updater.aws_lambda.lambda_api import LambdaAPI from game_updater.logger import logger - from game_updater.scraper.service import scrap_games + from game_updater.scraper.service import update_games from game_updater.steam.steam_api import SteamAPI def scrap_games_job(lambda_api: LambdaAPI): logger.info("-- scrap game job start -- ") - scrap_games(SteamAPI(), lambda_api) + update_games(SteamAPI(), lambda_api) logger.info("-- scrap game job end -- ") diff --git a/aws_lambdas/game_updater/scraper/alias.py b/aws_lambdas/game_updater/scraper/alias.py new file mode 100644 index 0000000..a8f7804 --- /dev/null +++ b/aws_lambdas/game_updater/scraper/alias.py @@ -0,0 +1,39 @@ +from collections import defaultdict +from collections.abc import Sequence + + +from ..igdb import igdb_api + + +def _scrap_igdb_game_ids(steam_ids: Sequence[int]) -> dict[int, int]: + steam_games = igdb_api.get_steam_games_batch(steam_ids) + return {int(g["uid"]): g["game"] for g in steam_games if "game" in g} + + +def _scrap_alternative_name_ids(igdb_game_ids: Sequence[int]) -> list[int]: + games = igdb_api.get_games_batch(igdb_game_ids) + ids: list[int] = [] + for g in games: + if "alternative_names" in g: + ids.extend(g["alternative_names"]) + + return ids + + +def _scrap_alternative_names(alternative_name_ids: Sequence[int]) -> dict[int, list[str]]: + alternative_names = igdb_api.get_alternative_names_batch(alternative_name_ids) + + game2alter_names: dict[int, list[str]] = defaultdict(list) + + for a in alternative_names: + game2alter_names[a["game"]].append(a["name"]) + + return game2alter_names + + +def scrap_aliases(steam_ids: Sequence[int]) -> dict[int, list[str]]: + igdb_game_ids = _scrap_igdb_game_ids(steam_ids) + alter_name_ids = _scrap_alternative_name_ids(list(igdb_game_ids.values())) + alter_names = _scrap_alternative_names(alter_name_ids) + + return {steam_id: alter_names[igdb_id] for steam_id, igdb_id in igdb_game_ids.items()} diff --git a/aws_lambdas/game_updater/scraper/game.py b/aws_lambdas/game_updater/scraper/game.py index 4fb81dd..0ea2551 100644 --- a/aws_lambdas/game_updater/scraper/game.py +++ b/aws_lambdas/game_updater/scraper/game.py @@ -1,10 +1,9 @@ import random from typing import Sequence -from ..aws_lambda.model import SaveGame from ..config import setting from ..logger import logger -from ..protocols import LambdaAPI, SteamAPI +from ..protocols import SteamAPI from .model import Game @@ -49,7 +48,7 @@ def _filter_games(games: Sequence[Game]) -> list[Game]: return filtered -def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: +def scrap_games(steam_api: SteamAPI) -> list[Game]: # get all steam games logger.info("getting all steam games") games = _scrap_all_steam_games(steam_api, setting.WORKER_CNT) @@ -62,11 +61,4 @@ def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: logger.info("game cnt: %s", len(games)) logger.debug("sample games: %s", random.sample(games, min(len(games), 5))) - # save games - logger.info("saving games") - games = lambda_api.save_games( - [ - SaveGame(steam_id=g.steam_id, name=g.name, released_at=g.released_at, kr_name=g.kr_name, genres=g.genres) - for g in games - ] - ) + return games diff --git a/aws_lambdas/game_updater/scraper/model.py b/aws_lambdas/game_updater/scraper/model.py index cc94ba1..d6ee7f5 100644 --- a/aws_lambdas/game_updater/scraper/model.py +++ b/aws_lambdas/game_updater/scraper/model.py @@ -1,12 +1,11 @@ -from typing import Optional, Sequence +from typing import Sequence -from pydantic import BaseModel, Field +from pydantic import BaseModel class Game(BaseModel): steam_id: int name: str - kr_name: Optional[str] = Field(default=None) released_at: float genres: Sequence[str] tags: Sequence[str] diff --git a/aws_lambdas/game_updater/scraper/service.py b/aws_lambdas/game_updater/scraper/service.py index 17d26d5..a86e90c 100644 --- a/aws_lambdas/game_updater/scraper/service.py +++ b/aws_lambdas/game_updater/scraper/service.py @@ -1,6 +1,25 @@ +from .alias import scrap_aliases from ..protocols import LambdaAPI, SteamAPI from . import game as game_scraper +from ..logger import logger +from ..aws_lambda.model import SaveGame -def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: - game_scraper.scrap_games(steam_api, lambda_api) +def update_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: + games = game_scraper.scrap_games(steam_api) + aliases = scrap_aliases([game.steam_id for game in games]) + + # save games + logger.info("saving games") + lambda_api.save_games( + [ + SaveGame( + steam_id=g.steam_id, + name=g.name, + released_at=g.released_at, + genres=g.genres, + aliases=aliases[g.steam_id], + ) + for g in games + ] + ) diff --git a/aws_lambdas/tests/database_lambda/game/test_game_service.py b/aws_lambdas/tests/database_lambda/game/test_game_service.py index 6817974..ed6328a 100644 --- a/aws_lambdas/tests/database_lambda/game/test_game_service.py +++ b/aws_lambdas/tests/database_lambda/game/test_game_service.py @@ -22,16 +22,16 @@ def test_save_games은_입력한_게임을_저장해야_한다(session: Session, { "steam_id": 1, "name": "game1", - "kr_name": "게임1", "released_at": random_datetime().timestamp(), "genres": ["Adventure"], + "aliases": ["게임1", "game 1"], }, { "steam_id": 2, "name": "game2", - "kr_name": "게임2", "released_at": random_datetime().timestamp(), "genres": ["Adventure", "RPG"], + "aliases": ["게임2", "game 2"], }, ] save_games(games, session=session, es_client=es_client) @@ -51,16 +51,16 @@ def test_save_games은_이미_저장한_게임을_중복저장하지_않는다(s { "steam_id": 1, "name": "game1", - "kr_name": "게임1", "released_at": random_datetime().timestamp(), "genres": ["Adventure"], + "aliases": ["게임1", "game 1"], }, { "steam_id": 2, "name": "game2", - "kr_name": "게임2", "released_at": random_datetime().timestamp(), "genres": ["Adventure", "RPG"], + "aliases": ["게임2", "game 2"], }, ] @@ -83,32 +83,32 @@ def test_save_games은_이미_저장한_게임을_중복저장하지_않는다(s { "steam_id": 1, "name": "game1", - "kr_name": "게임1", "released_at": random_datetime().timestamp(), "genres": ["Adventure"], + "aliases": ["게임", "게임1", "game 1"], }, { "steam_id": 1, "name": "game2", - "kr_name": "게임2", "released_at": random_datetime().timestamp(), "genres": ["RPG", "Adventure"], + "aliases": ["게임", "게임2", "game 2"], }, ), ( { "steam_id": 1, "name": "game2", - "kr_name": "게임2", "released_at": random_datetime().timestamp(), "genres": ["Adventure", "RPG"], + "aliases": ["게임", "게임2", "game 2"], }, { "steam_id": 1, "name": "game1", - "kr_name": "게임1", "released_at": random_datetime().timestamp(), "genres": ["Adventure"], + "aliases": ["게임", "게임1", "game 1"], }, ), ), @@ -122,10 +122,28 @@ def test_save_games은_이미_저장한_게임은_업데이트_한다( # check rdb saved = session.scalars(select(Game)).one().to_dto() assert saved.name == after_game["name"] - assert saved.kr_name == after_game["kr_name"] + assert set(saved.aliases) == set(after_game["aliases"]) assert set(saved.genres) == set(after_game["genres"]) # check es docs = search_game_docs(es_client) assert len(docs) == 1 assert docs[0]["_source"]["name"] == after_game["name"] + + +def test_save_games은_별칭을_저장한다(session: Session, es_client: Elasticsearch): + game: SaveGame = { + "steam_id": 1, + "name": "game1", + "released_at": random_datetime().timestamp(), + "genres": ["Adventure"], + "aliases": ["게임1", "game 1"], + } + + save_games([game], session=session, es_client=es_client) + + # check es + docs = search_game_docs(es_client) + game_doc = docs[0]["_source"] + + assert set(game["aliases"]) == set(game_doc["aliases"]) diff --git a/aws_lambdas/tests/database_lambda/utils/game.py b/aws_lambdas/tests/database_lambda/utils/game.py index dbf4b08..e511de8 100644 --- a/aws_lambdas/tests/database_lambda/utils/game.py +++ b/aws_lambdas/tests/database_lambda/utils/game.py @@ -8,6 +8,7 @@ from database_lambda.es import GAME_INDEX from database_lambda.game.model import Game from database_lambda.genre.model import Genre +from database_lambda.alias.model import GameAlias from .genre import create_random_genre from .utils import random_datetime @@ -19,23 +20,29 @@ def create_random_game( session: Session, *, name: Optional[str] = None, - kr_name: Optional[str] = None, released_at: Optional[datetime] = None, genres: Optional[list[Genre]] = None, + aliases: Optional[list[str]] = None, ) -> Game: global steam_id_counter steam_id_counter += 1 if name is None: name = f"Game #{steam_id_counter}" - if kr_name is None: - kr_name = f"게임 #{steam_id_counter}" if released_at is None: released_at = random_datetime() if genres is None: genres = [create_random_genre(session) for _ in range(randint(0, 3))] - - game = Game(steam_id=steam_id_counter, name=name, kr_name=kr_name, genres=genres, released_at=released_at) + if aliases is None: + aliases = ["Alias #1", "Alias #2", "Alias #3"] + + game = Game( + steam_id=steam_id_counter, + name=name, + genres=genres, + aliases=[GameAlias(name=alias) for alias in aliases], + released_at=released_at, + ) session.add(game) session.commit() diff --git a/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py b/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py deleted file mode 100644 index 078ff0f..0000000 --- a/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest - -from game_updater.config import setting -from game_updater.scraper.game import scrap_games -from tests.game_updater.utils.mock_lambda_api import MockLambdaAPI -from tests.game_updater.utils.mock_steam_api import MockSteamAPI -from tests.game_updater.utils.steam import create_random_game - - -@pytest.fixture -def mock_steam_api() -> MockSteamAPI: - return MockSteamAPI() - - -def test_scrap_geams는_새_게임을_저장한다(mock_steam_api: MockSteamAPI): - mock_steam_api.prepare_mock_data() - lambda_api = MockLambdaAPI() - - scrap_games(mock_steam_api, lambda_api) - - assert len(lambda_api.games.values()) > 0 - - -def test_scrap_geams는_수익이_높은_게임만_저장한다(mock_steam_api: MockSteamAPI): - popular_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[]) - unpopular_game = create_random_game(revenue=setting.MIN_REVENUE - 1, tags=[]) - mock_steam_api.add_mock_game(popular_game) - mock_steam_api.add_mock_game(unpopular_game) - lambda_api = MockLambdaAPI() - - scrap_games(mock_steam_api, lambda_api) - - saved = list(lambda_api.games.values()) - assert len(saved) == 1 - assert saved[0].steam_id == popular_game["steam_id"] - - -@pytest.mark.parametrize("sex_tag", ("Sexual Content", "NSFW")) -def test_scrap_geams는_성적인_게임을_저장하지_않는다(mock_steam_api: MockSteamAPI, sex_tag: str): - sexual_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[sex_tag]) - mock_steam_api.add_mock_game(sexual_game) - lambda_api = MockLambdaAPI() - - scrap_games(mock_steam_api, lambda_api) - - saved = list(lambda_api.games.values()) - assert len(saved) == 0 diff --git a/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py b/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py new file mode 100644 index 0000000..8849545 --- /dev/null +++ b/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py @@ -0,0 +1,10 @@ +from game_updater.scraper.alias import scrap_aliases + + +COUNTER_STRIKE = 10 + + +def test_scrap_aliases는_별칭을_구한다(): + scraped = scrap_aliases([COUNTER_STRIKE]) + + assert len(scraped[COUNTER_STRIKE]) > 0 diff --git a/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py b/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py new file mode 100644 index 0000000..cbbea5d --- /dev/null +++ b/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py @@ -0,0 +1,40 @@ +import pytest + +from game_updater.config import setting +from game_updater.scraper.game import scrap_games +from tests.game_updater.utils.mock_steam_api import MockSteamAPI +from tests.game_updater.utils.steam import create_random_game + + +@pytest.fixture +def mock_steam_api() -> MockSteamAPI: + return MockSteamAPI() + + +def test_scrap_games는_게임을_구한다(mock_steam_api: MockSteamAPI): + mock_steam_api.prepare_mock_data() + + scraped = scrap_games(mock_steam_api) + assert len(scraped) > 0 + + +def test_scrap_games는_수익이_높은_게임만_구한다(mock_steam_api: MockSteamAPI): + popular_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[]) + unpopular_game = create_random_game(revenue=setting.MIN_REVENUE - 1, tags=[]) + mock_steam_api.add_mock_game(popular_game) + mock_steam_api.add_mock_game(unpopular_game) + + scraped = scrap_games(mock_steam_api) + + assert len(scraped) == 1 + assert scraped[0].steam_id == popular_game["steam_id"] + + +@pytest.mark.parametrize("sex_tag", ("Sexual Content", "NSFW")) +def test_scrap_games는_성적인_게임을_구하지_않는다(mock_steam_api: MockSteamAPI, sex_tag: str): + sexual_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[sex_tag]) + mock_steam_api.add_mock_game(sexual_game) + + scraped = scrap_games(mock_steam_api) + + assert len(scraped) == 0 diff --git a/aws_lambdas/tests/game_updater/steam/test_steam_api.py b/aws_lambdas/tests/game_updater/steam/test_steam_api.py deleted file mode 100644 index 0c3af4e..0000000 --- a/aws_lambdas/tests/game_updater/steam/test_steam_api.py +++ /dev/null @@ -1,44 +0,0 @@ -# import pytest - -# from game_updater.steam.exception import SteamAPINoContentsException -# from game_updater.steam.steam_api import SteamAPI - - -# @pytest.fixture -# def app_id() -> int: -# return 70 # half-life - - -# @pytest.fixture -# def no_detail_app_id() -> int: -# return 1599340 # Lost Ark - - -# @pytest.fixture -# def steam_api() -> SteamAPI: -# return SteamAPI() - - -# def test_get_feature_games(steam_api: SteamAPI): -# assert steam_api.get_feature_games() - - -# def test_get_game_screenshots(steam_api: SteamAPI, app_id: int): -# assert steam_api.get_game_screenshots(app_id) - - -# def test_get_game_details(steam_api: SteamAPI, app_id: int): -# assert steam_api.get_game_details(app_id) - - -# def test_get_game_details_kor(steam_api: SteamAPI, app_id: int): -# assert steam_api.get_game_details(app_id, "korean") - - -# def test_get_game_details_from_gamalytic(steam_api: SteamAPI, app_id: int): -# assert steam_api.get_game_details_from_gamalytic(app_id) - - -# def test_get_game_details_no_content(steam_api: SteamAPI, no_detail_app_id: int): -# with pytest.raises(SteamAPINoContentsException): -# steam_api.get_game_details(no_detail_app_id) diff --git a/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py b/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py deleted file mode 100644 index 10678c0..0000000 --- a/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py +++ /dev/null @@ -1,23 +0,0 @@ -from datetime import datetime -from typing import Sequence - -from game_updater.aws_lambda.model import Game, SaveGame -from game_updater.protocols import LambdaAPI - - -class MockLambdaAPI(LambdaAPI): - def __init__(self): - self.games: dict[int, Game] = {} - - def save_games(self, games: Sequence[SaveGame]): - for game in games: - self.games[game.steam_id] = Game( - id=game.steam_id, - steam_id=game.steam_id, - name=game.name, - kr_name=game.kr_name, - released_at=datetime.fromtimestamp(game.released_at), - genres=game.genres, - updated_at=datetime.utcnow(), - created_at=datetime.utcnow(), - ) diff --git a/aws_lambdas/tests/game_updater/utils/model.py b/aws_lambdas/tests/game_updater/utils/model.py deleted file mode 100644 index a7fb686..0000000 --- a/aws_lambdas/tests/game_updater/utils/model.py +++ /dev/null @@ -1,45 +0,0 @@ -from datetime import datetime -from typing import Optional - -from game_updater.aws_lambda.model import Game - -from .utils import random_datetime - -game_id_counter = 1 - - -def create_random_game( - *, - steam_id: Optional[int] = None, - name: Optional[str] = None, - kr_name: Optional[str] = None, - genres: Optional[list[str]] = None, - released_at: Optional[datetime] = None, -) -> Game: - global game_id_counter - game_id_counter += 1 - - if steam_id is None: - steam_id = game_id_counter - if name is None: - name = f"Game #{steam_id}" - if kr_name is None: - kr_name = f"게임 #{steam_id}" - if genres is None: - genres = [f"Genre #{steam_id}"] - if released_at is None: - released_at = random_datetime() - - created_at = datetime.utcnow() - updated_at = datetime.utcnow() - - return Game( - id=game_id_counter, - steam_id=steam_id, - name=name, - kr_name=kr_name, - created_at=created_at, - updated_at=updated_at, - genres=genres, - released_at=released_at, - ) diff --git a/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py b/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py new file mode 100644 index 0000000..1b08808 --- /dev/null +++ b/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py @@ -0,0 +1,40 @@ +"""0004-ADD-GAME-ALIAS-TABLE + +Revision ID: 5d90f00aa849 +Revises: 5ef83623bd0d +Create Date: 2024-03-11 18:05:54.036373 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '5d90f00aa849' +down_revision: Union[str, None] = '5ef83623bd0d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('game_alias', + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('game_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['game_id'], ['game.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('game_alias') + # ### end Alembic commands ### diff --git a/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py b/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py new file mode 100644 index 0000000..6751676 --- /dev/null +++ b/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py @@ -0,0 +1,33 @@ +"""0006-GAME-ALIAS-GAME-ID-NAME-UNIQUE + +Revision ID: cc7150130bcf +Revises: d8e0e8e12a4a +Create Date: 2024-03-12 01:40:10.343486 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = 'cc7150130bcf' +down_revision: Union[str, None] = 'd8e0e8e12a4a' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('name', table_name='game_alias') + op.create_unique_constraint(None, 'game_alias', ['game_id', 'name']) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'game_alias', type_='unique') + op.create_index('name', 'game_alias', ['name'], unique=True) + # ### end Alembic commands ### diff --git a/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py b/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py new file mode 100644 index 0000000..6d8608e --- /dev/null +++ b/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py @@ -0,0 +1,31 @@ +"""0005-REMOVE-GAME-KR-NAME-FIELD + +Revision ID: d8e0e8e12a4a +Revises: 5d90f00aa849 +Create Date: 2024-03-11 18:08:03.049988 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision: str = 'd8e0e8e12a4a' +down_revision: Union[str, None] = '5d90f00aa849' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('game', 'kr_name') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('game', sa.Column('kr_name', mysql.VARCHAR(length=255), nullable=True)) + # ### end Alembic commands ### diff --git a/backend/src/game/model.py b/backend/src/game/model.py index e6de2f0..6575036 100644 --- a/backend/src/game/model.py +++ b/backend/src/game/model.py @@ -5,6 +5,8 @@ from ..model import CreatedAtMixin, UpdatedAtMixin +# Genre + class GameGenreLink(SQLModel, table=True): __tablename__: str = "game_genre_link" @@ -23,6 +25,9 @@ class Genre(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True): name: str = Field(max_length=64, unique=True) +# Game + + class Game(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True): __tablename__: str = "game" @@ -30,8 +35,24 @@ class Game(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True): steam_id: int = Field(unique=True) name: str = Field(max_length=64) released_at: datetime = Field() - kr_name: str | None = Field(max_length=64) genres: list[Genre] = Relationship(link_model=GameGenreLink) + aliases: list["GameAlias"] = Relationship(sa_relationship_kwargs={"cascade": "all,delete-orphan"}) + + +# Game Alias + + +class GameAlias(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True): + __tablename__: str = "game_alias" + __table_args__ = (UniqueConstraint("game_id", "name"),) + + id: int | None = Field(default=None, primary_key=True) + name: str = Field(max_length=64) + + game_id: int | None = Field(default=None, foreign_key="game.id") + + +# Screenshot class GameScreenshot(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True): diff --git a/backend/src/game/schema.py b/backend/src/game/schema.py index 15137e1..5649869 100644 --- a/backend/src/game/schema.py +++ b/backend/src/game/schema.py @@ -5,7 +5,7 @@ class AutoCompleteName(BaseModel): name: str - locale_name: str | None + match: str class AutoCompleteNameResponse(BaseModel): diff --git a/backend/src/game/service.py b/backend/src/game/service.py index 502691f..2cf71b8 100644 --- a/backend/src/game/service.py +++ b/backend/src/game/service.py @@ -9,12 +9,7 @@ def __init__(self, es_client: AsyncElasticsearch) -> None: self._es_client = es_client async def auto_complete_name(self, query: str) -> list[AutoCompleteName]: - game_names = await self._search_game_name(query) - - return [AutoCompleteName(name=name, locale_name=None) for name in game_names] - - async def _search_game_name(self, query: str) -> list[str]: - game_names: list[str] = [] + auto_complete_names: list[AutoCompleteName] = [] res = await self._es_client.search( index=GAME_INDEX, body={ @@ -23,14 +18,27 @@ async def _search_game_name(self, query: str) -> list[str]: "should": [ {"match_phrase_prefix": {"q_name": {"query": query.lower()}}}, {"match_phrase_prefix": {"name": {"query": query.lower()}}}, + {"match_phrase_prefix": {"aliases": {"query": query.lower()}}}, ] } - } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": {"q_name": {}, "name": {}, "aliases": {}}, + }, }, ) for hit in res["hits"]["hits"]: - name = hit["_source"]["name"] - game_names.append(name) + name: str = hit["_source"]["name"] + + highlight = hit["highlight"] + if "q_name" in highlight or "name" in highlight: + match = name + elif "aliases" in highlight: + match = highlight["aliases"][0] + + auto_complete_names.append(AutoCompleteName(name=name, match=match)) - return game_names + return auto_complete_names diff --git a/backend/tests/integration/game/test_auto_complete_name.py b/backend/tests/integration/game/test_auto_complete_name.py index c033373..7f1bd91 100644 --- a/backend/tests/integration/game/test_auto_complete_name.py +++ b/backend/tests/integration/game/test_auto_complete_name.py @@ -18,8 +18,8 @@ def client() -> Generator[TestClient, Any, None]: yield client -def create_indexed_game(session: Session, es_client: Elasticsearch, name: str) -> Game: - game = create_random_game(session, name=name, kr_name="") +def create_indexed_game(session: Session, es_client: Elasticsearch, name: str, aliases: list[str] = []) -> Game: + game = create_random_game(session, name=name, aliases=aliases) index_game(es_client, game) return game @@ -46,7 +46,27 @@ def test_auto_complete_game_name( assert res.status_code == status.HTTP_200_OK res_json = res.json() - assert res_json["games"] == [{"name": saved_game.name, "locale_name": None}] + assert res_json["games"] == [{"name": saved_game.name, "match": saved_game.name}] + + +@pytest.mark.parametrize( + ("game_name", "alias", "query"), + ( + ("NieR:Automata", "니어오토마타", "니어"), + ("NieR:Automata", "니어오토마타", "니어오토마타"), + ("NieR:Automata", "오토마타", "오토"), + ), +) +def test_auto_complete_game_name_by_alias( + client: TestClient, session: Session, es_client: Elasticsearch, game_name: str, alias: str, query: str +): + saved_game = create_indexed_game(session, es_client, game_name, [alias]) + + res = client.get(f"/game/auto_complete_name?query={query}") + assert res.status_code == status.HTTP_200_OK + + res_json = res.json() + assert res_json["games"] == [{"name": saved_game.name, "match": alias}] def test_auto_complete_for_multiple_games(client: TestClient, session: Session, es_client: Elasticsearch): diff --git a/backend/tests/utils/game.py b/backend/tests/utils/game.py index fe839d3..5571e71 100644 --- a/backend/tests/utils/game.py +++ b/backend/tests/utils/game.py @@ -5,29 +5,32 @@ from sqlmodel import Session from src.es import GAME_INDEX -from src.game.model import Game +from src.game.model import Game, GameAlias -from .utils import random_datetime, random_kr_string, random_name +from .utils import random_datetime, random_name steam_id_counter = 0 steam_id_lock = Lock() def create_random_game( - session: Session, *, name: str | None = None, kr_name: str | None = None, released_at: datetime | None = None + session: Session, *, name: str | None = None, released_at: datetime | None = None, aliases: list[str] = [] ) -> Game: global steam_id_counter if name is None: name = random_name() - if kr_name is None: - kr_name = random_kr_string() if released_at is None: released_at = random_datetime() with steam_id_lock: steam_id_counter += 1 - game = Game(steam_id=steam_id_counter, name=name, kr_name=kr_name, released_at=released_at) + game = Game( + steam_id=steam_id_counter, + name=name, + released_at=released_at, + aliases=[GameAlias(name=alias_name) for alias_name in aliases], + ) session.add(game) session.commit() @@ -38,4 +41,8 @@ def create_random_game( def index_game(es_client: Elasticsearch, game: Game): q_name = "".join([c if c.isalnum() else " " for c in game.name]) - es_client.index(index=GAME_INDEX, body={"q_name": q_name, "name": game.name, "id": game.id}, refresh=True) + es_client.index( + index=GAME_INDEX, + body={"q_name": q_name, "name": game.name, "aliases": [alias.name for alias in game.aliases], "id": game.id}, + refresh=True, + ) diff --git a/backend/tests/utils/utils.py b/backend/tests/utils/utils.py index 1cf4ece..6074fbd 100644 --- a/backend/tests/utils/utils.py +++ b/backend/tests/utils/utils.py @@ -11,10 +11,6 @@ def random_name() -> str: return faker.name() -def random_kr_string() -> str: - return faker["ko-KR"].name() # type: ignore - - def random_image_url() -> str: return faker.image_url() diff --git a/frontend/app/daily/[page]/autocomplete.tsx b/frontend/app/daily/[page]/autocomplete.tsx index 61ee618..1b4df03 100644 --- a/frontend/app/daily/[page]/autocomplete.tsx +++ b/frontend/app/daily/[page]/autocomplete.tsx @@ -4,6 +4,7 @@ import { autoCompleteGameName } from "@/utils/backend-api"; type CompleteName = { name: string + match: string } type FieldState = { @@ -36,7 +37,7 @@ export default function AutoCompleteGameName({onChangeGuessName}: {onChangeGuess }; }); - onChangeGuessName(key.toString()); + onChangeGuessName(key?.toString()); }; // Specify how each of the Autocomplete values should change when the input @@ -86,7 +87,7 @@ export default function AutoCompleteGameName({onChangeGuessName}: {onChangeGuess onSelectionChange={onSelectionChange} onKeyDown={(e: any) => e.continuePropagation()} > - {(item) => {item.name}} + {(item) => {item.name}

{item.match}

} ); } diff --git a/frontend/package.json b/frontend/package.json index 5db3218..5b39ed9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "gdet-frontend", - "version": "3", + "version": "4", "private": true, "scripts": { "dev": "next dev",