diff --git a/.github/workflows/test-lambda.yaml b/.github/workflows/test-lambda.yaml
index 5cba7fe..e62a802 100644
--- a/.github/workflows/test-lambda.yaml
+++ b/.github/workflows/test-lambda.yaml
@@ -47,3 +47,6 @@ jobs:
- name: test app
run: poetry run python -m pytest tests -c pytest.ini
working-directory: ${{env.working-directory}}
+ env:
+ IGDB_CLIENT_ID: ${{ secrets.IGDB_CLIENT_ID }}
+ IGDB_CLIENT_SECRET: ${{ secrets.IGDB_CLIENT_SECRET }}
diff --git a/aws_lambdas/database_lambda/alias/model.py b/aws_lambdas/database_lambda/alias/model.py
new file mode 100644
index 0000000..7cb7efc
--- /dev/null
+++ b/aws_lambdas/database_lambda/alias/model.py
@@ -0,0 +1,36 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from sqlalchemy import ForeignKey, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from ..model import Base, CreatedAtMixin, UpdatedAtMixin
+
+
+class GameAlias(CreatedAtMixin, UpdatedAtMixin, Base):
+ __tablename__ = "game_alias"
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ name: Mapped[str] = mapped_column(String(64))
+
+ game_id: Mapped[int] = mapped_column(ForeignKey("game.id"))
+
+ def __repr__(self) -> str:
+ return f"GameAlias(id={self.id}, name={self.name}, game_id={self.game_id}"
+
+ def to_dto(self) -> "GameAliasDto":
+ return GameAliasDto(
+ id=self.id,
+ name=self.name,
+ game_id=self.game_id,
+ created_at=self.created_at,
+ updated_at=self.updated_at,
+ )
+
+
+class GameAliasDto(BaseModel):
+ id: int
+ name: str
+ game_id: int
+ created_at: datetime
+ updated_at: datetime
diff --git a/aws_lambdas/database_lambda/game/es.py b/aws_lambdas/database_lambda/game/es.py
index 31ad5bd..34feceb 100644
--- a/aws_lambdas/database_lambda/game/es.py
+++ b/aws_lambdas/database_lambda/game/es.py
@@ -4,18 +4,20 @@
from elasticsearch import Elasticsearch, helpers
from ..es import GAME_INDEX
+from ..utils import is_aldecimal
from .model import Game
def _bulk_game_data(games: Iterable[Game]) -> Generator[dict[str, Any], Any, None]:
for game in games:
- q_name = "".join([c if c.isalnum() else " " for c in game.name])
+ q_name = "".join([c if is_aldecimal(c) else " " for c in game.name])
yield {
"_index": GAME_INDEX,
"_id": game.id,
"id": game.id,
"name": game.name,
"q_name": q_name,
+ "aliases": [alias.name for alias in game.aliases],
}
diff --git a/aws_lambdas/database_lambda/game/model.py b/aws_lambdas/database_lambda/game/model.py
index 95776f3..472b11a 100644
--- a/aws_lambdas/database_lambda/game/model.py
+++ b/aws_lambdas/database_lambda/game/model.py
@@ -1,10 +1,11 @@
from datetime import datetime
-from typing import Optional
from pydantic import BaseModel
from sqlalchemy import Column, ForeignKey, Integer, String, Table
from sqlalchemy.orm import Mapped, mapped_column, relationship
+from ..alias.model import GameAlias
+
from ..genre.model import Genre
from ..model import Base, CreatedAtMixin, UpdatedAtMixin
@@ -23,24 +24,21 @@ class Game(CreatedAtMixin, UpdatedAtMixin, Base):
id: Mapped[int] = mapped_column(primary_key=True)
steam_id: Mapped[int] = mapped_column(unique=True)
name: Mapped[str] = mapped_column(String(64))
- kr_name: Mapped[Optional[str]] = mapped_column(String(64))
released_at: Mapped[datetime] = mapped_column()
genres: Mapped[list[Genre]] = relationship(secondary=game_genre_link)
+ aliases: Mapped[list[GameAlias]] = relationship(cascade="all, delete-orphan")
def __repr__(self) -> str:
- return (
- f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, kr_name={self.kr_name},"
- f" released_at={self.released_at})"
- )
+ return f"Game(id={self.id}, steam_id={self.steam_id}, name={self.name}, released_at={self.released_at})"
def to_dto(self) -> "GameDto":
return GameDto(
id=self.id,
steam_id=self.steam_id,
name=self.name,
- kr_name=self.kr_name,
released_at=self.released_at,
genres=[g.name for g in self.genres],
+ aliases=[a.name for a in self.aliases],
created_at=self.created_at,
updated_at=self.updated_at,
)
@@ -50,8 +48,8 @@ class GameDto(BaseModel):
id: int
steam_id: int
name: str
- kr_name: Optional[str]
released_at: datetime
genres: list[str]
+ aliases: list[str]
created_at: datetime
updated_at: datetime
diff --git a/aws_lambdas/database_lambda/game/model_factory.py b/aws_lambdas/database_lambda/game/model_factory.py
index 929c1c0..18216e9 100644
--- a/aws_lambdas/database_lambda/game/model_factory.py
+++ b/aws_lambdas/database_lambda/game/model_factory.py
@@ -3,7 +3,9 @@
from sqlalchemy.orm import Session
+from ..alias.model import GameAlias
from ..genre.model_factory import to_models as to_genre_models
+from ..utils import is_aldecimal
from . import repository
from .model import Game
from .schema import STEAM_ID, SaveGame
@@ -13,9 +15,9 @@ def _create_model(session: Session, game: SaveGame) -> Game:
return Game(
steam_id=game["steam_id"],
name=game["name"],
- kr_name=game["kr_name"],
released_at=datetime.fromtimestamp(game["released_at"]),
genres=to_genre_models(session, game["genres"]),
+ aliases=[],
)
@@ -33,13 +35,36 @@ def _attach_models(session: Session, models: dict[STEAM_ID, Game]):
for game in saved:
query = models[game.steam_id]
game.name = query.name
- game.kr_name = query.kr_name
game.genres = query.genres
models[game.steam_id] = game
+def _update_aliases(session: Session, game: Game, aliases: set[str]):
+ # remove aliases
+ existed_aliases: list[str] = []
+ for game_alias in game.aliases:
+ if game_alias.name not in aliases:
+ session.delete(game_alias)
+ else:
+ existed_aliases.append(game_alias.name)
+
+ # add aliases
+ for alias_name in aliases:
+ if alias_name not in existed_aliases:
+ game.aliases.append(GameAlias(name=alias_name))
+
+
+def _allow_alias_name(alias_name: str):
+ return all(c == " " or is_aldecimal(c) for c in alias_name)
+
+
def to_models(session: Session, games: Iterable[SaveGame]) -> list[Game]:
models = _create_models(session, games)
_attach_models(session, models)
+ for game in games:
+ model = models[game["steam_id"]]
+ aliases = set(alias_name.lower() for alias_name in game["aliases"] if _allow_alias_name(alias_name))
+ _update_aliases(session, model, aliases)
+
return list(models.values())
diff --git a/aws_lambdas/database_lambda/game/schema.py b/aws_lambdas/database_lambda/game/schema.py
index 9a10609..7eae637 100644
--- a/aws_lambdas/database_lambda/game/schema.py
+++ b/aws_lambdas/database_lambda/game/schema.py
@@ -1,5 +1,5 @@
from collections.abc import Sequence
-from typing import Optional, TypedDict
+from typing import TypedDict
STEAM_ID = int
@@ -7,6 +7,6 @@
class SaveGame(TypedDict):
steam_id: int
name: str
- kr_name: Optional[str]
+ aliases: Sequence[str]
released_at: float
genres: Sequence[str]
diff --git a/aws_lambdas/database_lambda/utils.py b/aws_lambdas/database_lambda/utils.py
new file mode 100644
index 0000000..b70ed2d
--- /dev/null
+++ b/aws_lambdas/database_lambda/utils.py
@@ -0,0 +1,2 @@
+def is_aldecimal(s: str) -> bool:
+ return all(c.isdecimal() or c.isalpha() for c in s)
diff --git a/aws_lambdas/game_updater/aws_lambda/model.py b/aws_lambdas/game_updater/aws_lambda/model.py
index 8857f70..a698526 100644
--- a/aws_lambdas/game_updater/aws_lambda/model.py
+++ b/aws_lambdas/game_updater/aws_lambda/model.py
@@ -1,23 +1,11 @@
-from datetime import datetime
-from typing import Optional, Sequence
+from typing import Sequence
from pydantic import BaseModel
-class Game(BaseModel):
- id: int
- steam_id: int
- name: str
- kr_name: Optional[str]
- released_at: datetime
- genres: Sequence[str]
- updated_at: datetime
- created_at: datetime
-
-
class SaveGame(BaseModel):
steam_id: int
name: str
- kr_name: Optional[str]
released_at: float
genres: Sequence[str]
+ aliases: Sequence[str]
diff --git a/aws_lambdas/game_updater/config.py b/aws_lambdas/game_updater/config.py
index e4a09aa..3d61506 100644
--- a/aws_lambdas/game_updater/config.py
+++ b/aws_lambdas/game_updater/config.py
@@ -2,11 +2,15 @@
class Config(BaseSettings):
+ DATABASE_LAMBDA_NAME: str = "database"
+
WORKER_CNT: int = 10
MIN_REVENUE: int = 10000000 # 10M
- DATABASE_LAMBDA_NAME: str = "database"
- model_config = SettingsConfigDict(env_file=".scraper.env", env_file_encoding="utf-8")
+ IGDB_CLIENT_ID: str
+ IGDB_CLIENT_SECRET: str
+
+ model_config = SettingsConfigDict(env_file=".game_updater.env", env_file_encoding="utf-8")
setting = Config() # type: ignore
diff --git a/aws_lambdas/game_updater/igdb/igdb_api.py b/aws_lambdas/game_updater/igdb/igdb_api.py
new file mode 100644
index 0000000..a65ca40
--- /dev/null
+++ b/aws_lambdas/game_updater/igdb/igdb_api.py
@@ -0,0 +1,121 @@
+from collections.abc import Iterable, Sequence
+from functools import cache, wraps
+
+from .model import IGDBAlternativeName, IGDBExternalGames, IGDBGame
+from ..config import setting
+
+import requests
+
+STEAM_CATEGORY = 1
+MAX_LIMIT = 500
+
+
+def batch(size: int):
+ def batch_decorator(func):
+
+ @wraps(func)
+ def wrapper(_input: Sequence[int], **kwargs):
+ outputs = []
+
+ for srt_idx in range(0, len(_input), size):
+ batch_input = _input[srt_idx : srt_idx + size]
+ batch_output = func(batch_input, **kwargs)
+ outputs.extend(batch_output)
+
+ return outputs
+
+ return wrapper
+
+ return batch_decorator
+
+
+@cache
+def _get_token():
+ res = requests.post(
+ "https://id.twitch.tv/oauth2/token"
+ f"?client_id={setting.IGDB_CLIENT_ID}&client_secret={setting.IGDB_CLIENT_SECRET}&grant_type=client_credentials",
+ )
+ token = res.json()["access_token"]
+ return token
+
+
+def get_external_games(steam_ids: Iterable, category: int, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]:
+ # category,checksum,countries,created_at,game,media,name,platform,uid,updated_at,url,year;
+
+ uids = ",".join([f'"{id}"' for id in steam_ids])
+ token = _get_token()
+
+ response = requests.post(
+ "https://api.igdb.com/v4/external_games",
+ **{
+ "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+ "data": f"fields game,uid; where uid=({uids}) & category={STEAM_CATEGORY}; limit {limit};",
+ },
+ )
+ response.raise_for_status()
+
+ return response.json()
+
+
+def get_steam_games(steam_ids: Iterable, limit: int = MAX_LIMIT) -> list[IGDBExternalGames]:
+ return get_external_games(steam_ids, STEAM_CATEGORY, limit)
+
+
+def get_games(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBGame]:
+ # checksum,cover,created_at,game,name,region,updated_at;
+
+ ids_ = ",".join(map(str, ids))
+ token = _get_token()
+
+ response = requests.post(
+ "https://api.igdb.com/v4/games",
+ **{
+ "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+ "data": f"fields alternative_names; where id=({ids_}); limit {limit};",
+ },
+ )
+ response.raise_for_status()
+
+ return response.json()
+
+
+def get_alternative_names(ids: Iterable[int], limit: int = MAX_LIMIT) -> list[IGDBAlternativeName]:
+ # checksum,cover,created_at,game,name,region,updated_at;
+
+ ids_ = ",".join(map(str, ids))
+ token = _get_token()
+
+ response = requests.post(
+ "https://api.igdb.com/v4/alternative_names",
+ **{
+ "headers": {"Client-ID": setting.IGDB_CLIENT_ID, "Authorization": f"Bearer {token}"},
+ "data": f"fields name,game; where id=({ids_}); limit {limit};",
+ },
+ )
+ response.raise_for_status()
+
+ return response.json()
+
+
+@batch(MAX_LIMIT)
+def get_steam_games_batch(steam_ids: Sequence[int]) -> list[IGDBExternalGames]:
+ if len(steam_ids) == 0:
+ return []
+
+ return get_steam_games(steam_ids=steam_ids)
+
+
+@batch(MAX_LIMIT)
+def get_games_batch(game_ids: Sequence[int]) -> list[IGDBGame]:
+ if len(game_ids) == 0:
+ return []
+
+ return get_games(ids=game_ids)
+
+
+@batch(MAX_LIMIT)
+def get_alternative_names_batch(alternative_name_ids: Sequence[int]) -> list[IGDBAlternativeName]:
+ if len(alternative_name_ids) == 0:
+ return []
+
+ return get_alternative_names(ids=alternative_name_ids)
diff --git a/aws_lambdas/game_updater/igdb/model.py b/aws_lambdas/game_updater/igdb/model.py
new file mode 100644
index 0000000..6ede39e
--- /dev/null
+++ b/aws_lambdas/game_updater/igdb/model.py
@@ -0,0 +1,18 @@
+from typing import TypedDict
+
+
+class IGDBExternalGames(TypedDict):
+ id: int
+ uid: str
+ game: int
+
+
+class IGDBGame(TypedDict):
+ id: int
+ alternative_names: list[int]
+
+
+class IGDBAlternativeName(TypedDict):
+ id: int
+ name: str
+ game: int
diff --git a/aws_lambdas/game_updater/lambda_func.py b/aws_lambdas/game_updater/lambda_func.py
index 20ddfe3..2c7ecf7 100644
--- a/aws_lambdas/game_updater/lambda_func.py
+++ b/aws_lambdas/game_updater/lambda_func.py
@@ -5,13 +5,13 @@ def lambda_handler(event: Any, context: Any):
from game_updater.aws_lambda.lambda_api import LambdaAPI
from game_updater.logger import logger
- from game_updater.scraper.service import scrap_games
+ from game_updater.scraper.service import update_games
from game_updater.steam.steam_api import SteamAPI
def scrap_games_job(lambda_api: LambdaAPI):
logger.info("-- scrap game job start -- ")
- scrap_games(SteamAPI(), lambda_api)
+ update_games(SteamAPI(), lambda_api)
logger.info("-- scrap game job end -- ")
diff --git a/aws_lambdas/game_updater/scraper/alias.py b/aws_lambdas/game_updater/scraper/alias.py
new file mode 100644
index 0000000..a8f7804
--- /dev/null
+++ b/aws_lambdas/game_updater/scraper/alias.py
@@ -0,0 +1,39 @@
+from collections import defaultdict
+from collections.abc import Sequence
+
+
+from ..igdb import igdb_api
+
+
+def _scrap_igdb_game_ids(steam_ids: Sequence[int]) -> dict[int, int]:
+ steam_games = igdb_api.get_steam_games_batch(steam_ids)
+ return {int(g["uid"]): g["game"] for g in steam_games if "game" in g}
+
+
+def _scrap_alternative_name_ids(igdb_game_ids: Sequence[int]) -> list[int]:
+ games = igdb_api.get_games_batch(igdb_game_ids)
+ ids: list[int] = []
+ for g in games:
+ if "alternative_names" in g:
+ ids.extend(g["alternative_names"])
+
+ return ids
+
+
+def _scrap_alternative_names(alternative_name_ids: Sequence[int]) -> dict[int, list[str]]:
+ alternative_names = igdb_api.get_alternative_names_batch(alternative_name_ids)
+
+ game2alter_names: dict[int, list[str]] = defaultdict(list)
+
+ for a in alternative_names:
+ game2alter_names[a["game"]].append(a["name"])
+
+ return game2alter_names
+
+
+def scrap_aliases(steam_ids: Sequence[int]) -> dict[int, list[str]]:
+ igdb_game_ids = _scrap_igdb_game_ids(steam_ids)
+ alter_name_ids = _scrap_alternative_name_ids(list(igdb_game_ids.values()))
+ alter_names = _scrap_alternative_names(alter_name_ids)
+
+ return {steam_id: alter_names[igdb_id] for steam_id, igdb_id in igdb_game_ids.items()}
diff --git a/aws_lambdas/game_updater/scraper/game.py b/aws_lambdas/game_updater/scraper/game.py
index 4fb81dd..0ea2551 100644
--- a/aws_lambdas/game_updater/scraper/game.py
+++ b/aws_lambdas/game_updater/scraper/game.py
@@ -1,10 +1,9 @@
import random
from typing import Sequence
-from ..aws_lambda.model import SaveGame
from ..config import setting
from ..logger import logger
-from ..protocols import LambdaAPI, SteamAPI
+from ..protocols import SteamAPI
from .model import Game
@@ -49,7 +48,7 @@ def _filter_games(games: Sequence[Game]) -> list[Game]:
return filtered
-def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None:
+def scrap_games(steam_api: SteamAPI) -> list[Game]:
# get all steam games
logger.info("getting all steam games")
games = _scrap_all_steam_games(steam_api, setting.WORKER_CNT)
@@ -62,11 +61,4 @@ def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None:
logger.info("game cnt: %s", len(games))
logger.debug("sample games: %s", random.sample(games, min(len(games), 5)))
- # save games
- logger.info("saving games")
- games = lambda_api.save_games(
- [
- SaveGame(steam_id=g.steam_id, name=g.name, released_at=g.released_at, kr_name=g.kr_name, genres=g.genres)
- for g in games
- ]
- )
+ return games
diff --git a/aws_lambdas/game_updater/scraper/model.py b/aws_lambdas/game_updater/scraper/model.py
index cc94ba1..d6ee7f5 100644
--- a/aws_lambdas/game_updater/scraper/model.py
+++ b/aws_lambdas/game_updater/scraper/model.py
@@ -1,12 +1,11 @@
-from typing import Optional, Sequence
+from typing import Sequence
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
class Game(BaseModel):
steam_id: int
name: str
- kr_name: Optional[str] = Field(default=None)
released_at: float
genres: Sequence[str]
tags: Sequence[str]
diff --git a/aws_lambdas/game_updater/scraper/service.py b/aws_lambdas/game_updater/scraper/service.py
index 17d26d5..a86e90c 100644
--- a/aws_lambdas/game_updater/scraper/service.py
+++ b/aws_lambdas/game_updater/scraper/service.py
@@ -1,6 +1,25 @@
+from .alias import scrap_aliases
from ..protocols import LambdaAPI, SteamAPI
from . import game as game_scraper
+from ..logger import logger
+from ..aws_lambda.model import SaveGame
-def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None:
- game_scraper.scrap_games(steam_api, lambda_api)
+def update_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None:
+ games = game_scraper.scrap_games(steam_api)
+ aliases = scrap_aliases([game.steam_id for game in games])
+
+ # save games
+ logger.info("saving games")
+ lambda_api.save_games(
+ [
+ SaveGame(
+ steam_id=g.steam_id,
+ name=g.name,
+ released_at=g.released_at,
+ genres=g.genres,
+ aliases=aliases[g.steam_id],
+ )
+ for g in games
+ ]
+ )
diff --git a/aws_lambdas/tests/database_lambda/game/test_game_service.py b/aws_lambdas/tests/database_lambda/game/test_game_service.py
index 6817974..ed6328a 100644
--- a/aws_lambdas/tests/database_lambda/game/test_game_service.py
+++ b/aws_lambdas/tests/database_lambda/game/test_game_service.py
@@ -22,16 +22,16 @@ def test_save_games은_입력한_게임을_저장해야_한다(session: Session,
{
"steam_id": 1,
"name": "game1",
- "kr_name": "게임1",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure"],
+ "aliases": ["게임1", "game 1"],
},
{
"steam_id": 2,
"name": "game2",
- "kr_name": "게임2",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure", "RPG"],
+ "aliases": ["게임2", "game 2"],
},
]
save_games(games, session=session, es_client=es_client)
@@ -51,16 +51,16 @@ def test_save_games은_이미_저장한_게임을_중복저장하지_않는다(s
{
"steam_id": 1,
"name": "game1",
- "kr_name": "게임1",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure"],
+ "aliases": ["게임1", "game 1"],
},
{
"steam_id": 2,
"name": "game2",
- "kr_name": "게임2",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure", "RPG"],
+ "aliases": ["게임2", "game 2"],
},
]
@@ -83,32 +83,32 @@ def test_save_games은_이미_저장한_게임을_중복저장하지_않는다(s
{
"steam_id": 1,
"name": "game1",
- "kr_name": "게임1",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure"],
+ "aliases": ["게임", "게임1", "game 1"],
},
{
"steam_id": 1,
"name": "game2",
- "kr_name": "게임2",
"released_at": random_datetime().timestamp(),
"genres": ["RPG", "Adventure"],
+ "aliases": ["게임", "게임2", "game 2"],
},
),
(
{
"steam_id": 1,
"name": "game2",
- "kr_name": "게임2",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure", "RPG"],
+ "aliases": ["게임", "게임2", "game 2"],
},
{
"steam_id": 1,
"name": "game1",
- "kr_name": "게임1",
"released_at": random_datetime().timestamp(),
"genres": ["Adventure"],
+ "aliases": ["게임", "게임1", "game 1"],
},
),
),
@@ -122,10 +122,28 @@ def test_save_games은_이미_저장한_게임은_업데이트_한다(
# check rdb
saved = session.scalars(select(Game)).one().to_dto()
assert saved.name == after_game["name"]
- assert saved.kr_name == after_game["kr_name"]
+ assert set(saved.aliases) == set(after_game["aliases"])
assert set(saved.genres) == set(after_game["genres"])
# check es
docs = search_game_docs(es_client)
assert len(docs) == 1
assert docs[0]["_source"]["name"] == after_game["name"]
+
+
+def test_save_games은_별칭을_저장한다(session: Session, es_client: Elasticsearch):
+ game: SaveGame = {
+ "steam_id": 1,
+ "name": "game1",
+ "released_at": random_datetime().timestamp(),
+ "genres": ["Adventure"],
+ "aliases": ["게임1", "game 1"],
+ }
+
+ save_games([game], session=session, es_client=es_client)
+
+ # check es
+ docs = search_game_docs(es_client)
+ game_doc = docs[0]["_source"]
+
+ assert set(game["aliases"]) == set(game_doc["aliases"])
diff --git a/aws_lambdas/tests/database_lambda/utils/game.py b/aws_lambdas/tests/database_lambda/utils/game.py
index dbf4b08..e511de8 100644
--- a/aws_lambdas/tests/database_lambda/utils/game.py
+++ b/aws_lambdas/tests/database_lambda/utils/game.py
@@ -8,6 +8,7 @@
from database_lambda.es import GAME_INDEX
from database_lambda.game.model import Game
from database_lambda.genre.model import Genre
+from database_lambda.alias.model import GameAlias
from .genre import create_random_genre
from .utils import random_datetime
@@ -19,23 +20,29 @@ def create_random_game(
session: Session,
*,
name: Optional[str] = None,
- kr_name: Optional[str] = None,
released_at: Optional[datetime] = None,
genres: Optional[list[Genre]] = None,
+ aliases: Optional[list[str]] = None,
) -> Game:
global steam_id_counter
steam_id_counter += 1
if name is None:
name = f"Game #{steam_id_counter}"
- if kr_name is None:
- kr_name = f"게임 #{steam_id_counter}"
if released_at is None:
released_at = random_datetime()
if genres is None:
genres = [create_random_genre(session) for _ in range(randint(0, 3))]
-
- game = Game(steam_id=steam_id_counter, name=name, kr_name=kr_name, genres=genres, released_at=released_at)
+ if aliases is None:
+ aliases = ["Alias #1", "Alias #2", "Alias #3"]
+
+ game = Game(
+ steam_id=steam_id_counter,
+ name=name,
+ genres=genres,
+ aliases=[GameAlias(name=alias) for alias in aliases],
+ released_at=released_at,
+ )
session.add(game)
session.commit()
diff --git a/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py b/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py
deleted file mode 100644
index 078ff0f..0000000
--- a/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import pytest
-
-from game_updater.config import setting
-from game_updater.scraper.game import scrap_games
-from tests.game_updater.utils.mock_lambda_api import MockLambdaAPI
-from tests.game_updater.utils.mock_steam_api import MockSteamAPI
-from tests.game_updater.utils.steam import create_random_game
-
-
-@pytest.fixture
-def mock_steam_api() -> MockSteamAPI:
- return MockSteamAPI()
-
-
-def test_scrap_geams는_새_게임을_저장한다(mock_steam_api: MockSteamAPI):
- mock_steam_api.prepare_mock_data()
- lambda_api = MockLambdaAPI()
-
- scrap_games(mock_steam_api, lambda_api)
-
- assert len(lambda_api.games.values()) > 0
-
-
-def test_scrap_geams는_수익이_높은_게임만_저장한다(mock_steam_api: MockSteamAPI):
- popular_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[])
- unpopular_game = create_random_game(revenue=setting.MIN_REVENUE - 1, tags=[])
- mock_steam_api.add_mock_game(popular_game)
- mock_steam_api.add_mock_game(unpopular_game)
- lambda_api = MockLambdaAPI()
-
- scrap_games(mock_steam_api, lambda_api)
-
- saved = list(lambda_api.games.values())
- assert len(saved) == 1
- assert saved[0].steam_id == popular_game["steam_id"]
-
-
-@pytest.mark.parametrize("sex_tag", ("Sexual Content", "NSFW"))
-def test_scrap_geams는_성적인_게임을_저장하지_않는다(mock_steam_api: MockSteamAPI, sex_tag: str):
- sexual_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[sex_tag])
- mock_steam_api.add_mock_game(sexual_game)
- lambda_api = MockLambdaAPI()
-
- scrap_games(mock_steam_api, lambda_api)
-
- saved = list(lambda_api.games.values())
- assert len(saved) == 0
diff --git a/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py b/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py
new file mode 100644
index 0000000..8849545
--- /dev/null
+++ b/aws_lambdas/tests/game_updater/scraper/test_scrap_aliases.py
@@ -0,0 +1,10 @@
+from game_updater.scraper.alias import scrap_aliases
+
+
+COUNTER_STRIKE = 10
+
+
+def test_scrap_aliases는_별칭을_구한다():
+ scraped = scrap_aliases([COUNTER_STRIKE])
+
+ assert len(scraped[COUNTER_STRIKE]) > 0
diff --git a/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py b/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py
new file mode 100644
index 0000000..cbbea5d
--- /dev/null
+++ b/aws_lambdas/tests/game_updater/scraper/test_scrap_game.py
@@ -0,0 +1,40 @@
+import pytest
+
+from game_updater.config import setting
+from game_updater.scraper.game import scrap_games
+from tests.game_updater.utils.mock_steam_api import MockSteamAPI
+from tests.game_updater.utils.steam import create_random_game
+
+
+@pytest.fixture
+def mock_steam_api() -> MockSteamAPI:
+ return MockSteamAPI()
+
+
+def test_scrap_games는_게임을_구한다(mock_steam_api: MockSteamAPI):
+ mock_steam_api.prepare_mock_data()
+
+ scraped = scrap_games(mock_steam_api)
+ assert len(scraped) > 0
+
+
+def test_scrap_games는_수익이_높은_게임만_구한다(mock_steam_api: MockSteamAPI):
+ popular_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[])
+ unpopular_game = create_random_game(revenue=setting.MIN_REVENUE - 1, tags=[])
+ mock_steam_api.add_mock_game(popular_game)
+ mock_steam_api.add_mock_game(unpopular_game)
+
+ scraped = scrap_games(mock_steam_api)
+
+ assert len(scraped) == 1
+ assert scraped[0].steam_id == popular_game["steam_id"]
+
+
+@pytest.mark.parametrize("sex_tag", ("Sexual Content", "NSFW"))
+def test_scrap_games는_성적인_게임을_구하지_않는다(mock_steam_api: MockSteamAPI, sex_tag: str):
+ sexual_game = create_random_game(revenue=setting.MIN_REVENUE, tags=[sex_tag])
+ mock_steam_api.add_mock_game(sexual_game)
+
+ scraped = scrap_games(mock_steam_api)
+
+ assert len(scraped) == 0
diff --git a/aws_lambdas/tests/game_updater/steam/test_steam_api.py b/aws_lambdas/tests/game_updater/steam/test_steam_api.py
deleted file mode 100644
index 0c3af4e..0000000
--- a/aws_lambdas/tests/game_updater/steam/test_steam_api.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# import pytest
-
-# from game_updater.steam.exception import SteamAPINoContentsException
-# from game_updater.steam.steam_api import SteamAPI
-
-
-# @pytest.fixture
-# def app_id() -> int:
-# return 70 # half-life
-
-
-# @pytest.fixture
-# def no_detail_app_id() -> int:
-# return 1599340 # Lost Ark
-
-
-# @pytest.fixture
-# def steam_api() -> SteamAPI:
-# return SteamAPI()
-
-
-# def test_get_feature_games(steam_api: SteamAPI):
-# assert steam_api.get_feature_games()
-
-
-# def test_get_game_screenshots(steam_api: SteamAPI, app_id: int):
-# assert steam_api.get_game_screenshots(app_id)
-
-
-# def test_get_game_details(steam_api: SteamAPI, app_id: int):
-# assert steam_api.get_game_details(app_id)
-
-
-# def test_get_game_details_kor(steam_api: SteamAPI, app_id: int):
-# assert steam_api.get_game_details(app_id, "korean")
-
-
-# def test_get_game_details_from_gamalytic(steam_api: SteamAPI, app_id: int):
-# assert steam_api.get_game_details_from_gamalytic(app_id)
-
-
-# def test_get_game_details_no_content(steam_api: SteamAPI, no_detail_app_id: int):
-# with pytest.raises(SteamAPINoContentsException):
-# steam_api.get_game_details(no_detail_app_id)
diff --git a/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py b/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py
deleted file mode 100644
index 10678c0..0000000
--- a/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from datetime import datetime
-from typing import Sequence
-
-from game_updater.aws_lambda.model import Game, SaveGame
-from game_updater.protocols import LambdaAPI
-
-
-class MockLambdaAPI(LambdaAPI):
- def __init__(self):
- self.games: dict[int, Game] = {}
-
- def save_games(self, games: Sequence[SaveGame]):
- for game in games:
- self.games[game.steam_id] = Game(
- id=game.steam_id,
- steam_id=game.steam_id,
- name=game.name,
- kr_name=game.kr_name,
- released_at=datetime.fromtimestamp(game.released_at),
- genres=game.genres,
- updated_at=datetime.utcnow(),
- created_at=datetime.utcnow(),
- )
diff --git a/aws_lambdas/tests/game_updater/utils/model.py b/aws_lambdas/tests/game_updater/utils/model.py
deleted file mode 100644
index a7fb686..0000000
--- a/aws_lambdas/tests/game_updater/utils/model.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from game_updater.aws_lambda.model import Game
-
-from .utils import random_datetime
-
-game_id_counter = 1
-
-
-def create_random_game(
- *,
- steam_id: Optional[int] = None,
- name: Optional[str] = None,
- kr_name: Optional[str] = None,
- genres: Optional[list[str]] = None,
- released_at: Optional[datetime] = None,
-) -> Game:
- global game_id_counter
- game_id_counter += 1
-
- if steam_id is None:
- steam_id = game_id_counter
- if name is None:
- name = f"Game #{steam_id}"
- if kr_name is None:
- kr_name = f"게임 #{steam_id}"
- if genres is None:
- genres = [f"Genre #{steam_id}"]
- if released_at is None:
- released_at = random_datetime()
-
- created_at = datetime.utcnow()
- updated_at = datetime.utcnow()
-
- return Game(
- id=game_id_counter,
- steam_id=steam_id,
- name=name,
- kr_name=kr_name,
- created_at=created_at,
- updated_at=updated_at,
- genres=genres,
- released_at=released_at,
- )
diff --git a/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py b/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py
new file mode 100644
index 0000000..1b08808
--- /dev/null
+++ b/backend/migration/versions/5d90f00aa849_0004_add_game_alias_table.py
@@ -0,0 +1,40 @@
+"""0004-ADD-GAME-ALIAS-TABLE
+
+Revision ID: 5d90f00aa849
+Revises: 5ef83623bd0d
+Create Date: 2024-03-11 18:05:54.036373
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = '5d90f00aa849'
+down_revision: Union[str, None] = '5ef83623bd0d'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('game_alias',
+ sa.Column('updated_at', sa.DateTime(), nullable=False),
+ sa.Column('created_at', sa.DateTime(), nullable=False),
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+ sa.Column('game_id', sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(['game_id'], ['game.id'], ),
+ sa.PrimaryKeyConstraint('id'),
+ sa.UniqueConstraint('name')
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('game_alias')
+ # ### end Alembic commands ###
diff --git a/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py b/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py
new file mode 100644
index 0000000..6751676
--- /dev/null
+++ b/backend/migration/versions/cc7150130bcf_0006_game_alias_game_id_name_unique.py
@@ -0,0 +1,33 @@
+"""0006-GAME-ALIAS-GAME-ID-NAME-UNIQUE
+
+Revision ID: cc7150130bcf
+Revises: d8e0e8e12a4a
+Create Date: 2024-03-12 01:40:10.343486
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'cc7150130bcf'
+down_revision: Union[str, None] = 'd8e0e8e12a4a'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_index('name', table_name='game_alias')
+ op.create_unique_constraint(None, 'game_alias', ['game_id', 'name'])
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_constraint(None, 'game_alias', type_='unique')
+ op.create_index('name', 'game_alias', ['name'], unique=True)
+ # ### end Alembic commands ###
diff --git a/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py b/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py
new file mode 100644
index 0000000..6d8608e
--- /dev/null
+++ b/backend/migration/versions/d8e0e8e12a4a_0005_remove_game_kr_name_field.py
@@ -0,0 +1,31 @@
+"""0005-REMOVE-GAME-KR-NAME-FIELD
+
+Revision ID: d8e0e8e12a4a
+Revises: 5d90f00aa849
+Create Date: 2024-03-11 18:08:03.049988
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision: str = 'd8e0e8e12a4a'
+down_revision: Union[str, None] = '5d90f00aa849'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('game', 'kr_name')
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('game', sa.Column('kr_name', mysql.VARCHAR(length=255), nullable=True))
+ # ### end Alembic commands ###
diff --git a/backend/src/game/model.py b/backend/src/game/model.py
index e6de2f0..6575036 100644
--- a/backend/src/game/model.py
+++ b/backend/src/game/model.py
@@ -5,6 +5,8 @@
from ..model import CreatedAtMixin, UpdatedAtMixin
+# Genre
+
class GameGenreLink(SQLModel, table=True):
__tablename__: str = "game_genre_link"
@@ -23,6 +25,9 @@ class Genre(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True):
name: str = Field(max_length=64, unique=True)
+# Game
+
+
class Game(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True):
__tablename__: str = "game"
@@ -30,8 +35,24 @@ class Game(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True):
steam_id: int = Field(unique=True)
name: str = Field(max_length=64)
released_at: datetime = Field()
- kr_name: str | None = Field(max_length=64)
genres: list[Genre] = Relationship(link_model=GameGenreLink)
+ aliases: list["GameAlias"] = Relationship(sa_relationship_kwargs={"cascade": "all,delete-orphan"})
+
+
+# Game Alias
+
+
+class GameAlias(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True):
+ __tablename__: str = "game_alias"
+ __table_args__ = (UniqueConstraint("game_id", "name"),)
+
+ id: int | None = Field(default=None, primary_key=True)
+ name: str = Field(max_length=64)
+
+ game_id: int | None = Field(default=None, foreign_key="game.id")
+
+
+# Screenshot
class GameScreenshot(CreatedAtMixin, UpdatedAtMixin, SQLModel, table=True):
diff --git a/backend/src/game/schema.py b/backend/src/game/schema.py
index 15137e1..5649869 100644
--- a/backend/src/game/schema.py
+++ b/backend/src/game/schema.py
@@ -5,7 +5,7 @@
class AutoCompleteName(BaseModel):
name: str
- locale_name: str | None
+ match: str
class AutoCompleteNameResponse(BaseModel):
diff --git a/backend/src/game/service.py b/backend/src/game/service.py
index 502691f..2cf71b8 100644
--- a/backend/src/game/service.py
+++ b/backend/src/game/service.py
@@ -9,12 +9,7 @@ def __init__(self, es_client: AsyncElasticsearch) -> None:
self._es_client = es_client
async def auto_complete_name(self, query: str) -> list[AutoCompleteName]:
- game_names = await self._search_game_name(query)
-
- return [AutoCompleteName(name=name, locale_name=None) for name in game_names]
-
- async def _search_game_name(self, query: str) -> list[str]:
- game_names: list[str] = []
+ auto_complete_names: list[AutoCompleteName] = []
res = await self._es_client.search(
index=GAME_INDEX,
body={
@@ -23,14 +18,27 @@ async def _search_game_name(self, query: str) -> list[str]:
"should": [
{"match_phrase_prefix": {"q_name": {"query": query.lower()}}},
{"match_phrase_prefix": {"name": {"query": query.lower()}}},
+ {"match_phrase_prefix": {"aliases": {"query": query.lower()}}},
]
}
- }
+ },
+ "highlight": {
+ "pre_tags": [""],
+ "post_tags": [""],
+ "fields": {"q_name": {}, "name": {}, "aliases": {}},
+ },
},
)
for hit in res["hits"]["hits"]:
- name = hit["_source"]["name"]
- game_names.append(name)
+ name: str = hit["_source"]["name"]
+
+ highlight = hit["highlight"]
+ if "q_name" in highlight or "name" in highlight:
+ match = name
+ elif "aliases" in highlight:
+ match = highlight["aliases"][0]
+
+ auto_complete_names.append(AutoCompleteName(name=name, match=match))
- return game_names
+ return auto_complete_names
diff --git a/backend/tests/integration/game/test_auto_complete_name.py b/backend/tests/integration/game/test_auto_complete_name.py
index c033373..7f1bd91 100644
--- a/backend/tests/integration/game/test_auto_complete_name.py
+++ b/backend/tests/integration/game/test_auto_complete_name.py
@@ -18,8 +18,8 @@ def client() -> Generator[TestClient, Any, None]:
yield client
-def create_indexed_game(session: Session, es_client: Elasticsearch, name: str) -> Game:
- game = create_random_game(session, name=name, kr_name="")
+def create_indexed_game(session: Session, es_client: Elasticsearch, name: str, aliases: list[str] = []) -> Game:
+ game = create_random_game(session, name=name, aliases=aliases)
index_game(es_client, game)
return game
@@ -46,7 +46,27 @@ def test_auto_complete_game_name(
assert res.status_code == status.HTTP_200_OK
res_json = res.json()
- assert res_json["games"] == [{"name": saved_game.name, "locale_name": None}]
+ assert res_json["games"] == [{"name": saved_game.name, "match": saved_game.name}]
+
+
+@pytest.mark.parametrize(
+ ("game_name", "alias", "query"),
+ (
+ ("NieR:Automata", "니어오토마타", "니어"),
+ ("NieR:Automata", "니어오토마타", "니어오토마타"),
+ ("NieR:Automata", "오토마타", "오토"),
+ ),
+)
+def test_auto_complete_game_name_by_alias(
+ client: TestClient, session: Session, es_client: Elasticsearch, game_name: str, alias: str, query: str
+):
+ saved_game = create_indexed_game(session, es_client, game_name, [alias])
+
+ res = client.get(f"/game/auto_complete_name?query={query}")
+ assert res.status_code == status.HTTP_200_OK
+
+ res_json = res.json()
+ assert res_json["games"] == [{"name": saved_game.name, "match": alias}]
def test_auto_complete_for_multiple_games(client: TestClient, session: Session, es_client: Elasticsearch):
diff --git a/backend/tests/utils/game.py b/backend/tests/utils/game.py
index fe839d3..5571e71 100644
--- a/backend/tests/utils/game.py
+++ b/backend/tests/utils/game.py
@@ -5,29 +5,32 @@
from sqlmodel import Session
from src.es import GAME_INDEX
-from src.game.model import Game
+from src.game.model import Game, GameAlias
-from .utils import random_datetime, random_kr_string, random_name
+from .utils import random_datetime, random_name
steam_id_counter = 0
steam_id_lock = Lock()
def create_random_game(
- session: Session, *, name: str | None = None, kr_name: str | None = None, released_at: datetime | None = None
+ session: Session, *, name: str | None = None, released_at: datetime | None = None, aliases: list[str] = []
) -> Game:
global steam_id_counter
if name is None:
name = random_name()
- if kr_name is None:
- kr_name = random_kr_string()
if released_at is None:
released_at = random_datetime()
with steam_id_lock:
steam_id_counter += 1
- game = Game(steam_id=steam_id_counter, name=name, kr_name=kr_name, released_at=released_at)
+ game = Game(
+ steam_id=steam_id_counter,
+ name=name,
+ released_at=released_at,
+ aliases=[GameAlias(name=alias_name) for alias_name in aliases],
+ )
session.add(game)
session.commit()
@@ -38,4 +41,8 @@ def create_random_game(
def index_game(es_client: Elasticsearch, game: Game):
q_name = "".join([c if c.isalnum() else " " for c in game.name])
- es_client.index(index=GAME_INDEX, body={"q_name": q_name, "name": game.name, "id": game.id}, refresh=True)
+ es_client.index(
+ index=GAME_INDEX,
+ body={"q_name": q_name, "name": game.name, "aliases": [alias.name for alias in game.aliases], "id": game.id},
+ refresh=True,
+ )
diff --git a/backend/tests/utils/utils.py b/backend/tests/utils/utils.py
index 1cf4ece..6074fbd 100644
--- a/backend/tests/utils/utils.py
+++ b/backend/tests/utils/utils.py
@@ -11,10 +11,6 @@ def random_name() -> str:
return faker.name()
-def random_kr_string() -> str:
- return faker["ko-KR"].name() # type: ignore
-
-
def random_image_url() -> str:
return faker.image_url()
diff --git a/frontend/app/daily/[page]/autocomplete.tsx b/frontend/app/daily/[page]/autocomplete.tsx
index 61ee618..1b4df03 100644
--- a/frontend/app/daily/[page]/autocomplete.tsx
+++ b/frontend/app/daily/[page]/autocomplete.tsx
@@ -4,6 +4,7 @@ import { autoCompleteGameName } from "@/utils/backend-api";
type CompleteName = {
name: string
+ match: string
}
type FieldState = {
@@ -36,7 +37,7 @@ export default function AutoCompleteGameName({onChangeGuessName}: {onChangeGuess
};
});
- onChangeGuessName(key.toString());
+ onChangeGuessName(key?.toString());
};
// Specify how each of the Autocomplete values should change when the input
@@ -86,7 +87,7 @@ export default function AutoCompleteGameName({onChangeGuessName}: {onChangeGuess
onSelectionChange={onSelectionChange}
onKeyDown={(e: any) => e.continuePropagation()}
>
- {(item) => {item.name}}
+ {(item) => {item.name}{item.match}
}
);
}
diff --git a/frontend/package.json b/frontend/package.json
index 5db3218..5b39ed9 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "gdet-frontend",
- "version": "3",
+ "version": "4",
"private": true,
"scripts": {
"dev": "next dev",