From 858dbbdeb9d9a5283e44666ba29b4b3259239c3b Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Fri, 2 Feb 2024 23:56:22 +0900 Subject: [PATCH 01/31] =?UTF-8?q?refactor:=20=EB=9E=8C=EB=8B=A4=20?= =?UTF-8?q?=EB=94=94=EB=A0=89=ED=84=B0=EB=A6=AC=20=EC=9D=B4=EB=A6=84=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/{private => database}/config.py | 0 data_scrapers/{private => database}/database.py | 0 data_scrapers/{private => database}/event.py | 0 data_scrapers/{private => database}/game/model.py | 0 data_scrapers/{private => database}/game/repository.py | 0 data_scrapers/{private => database}/game/service.py | 0 data_scrapers/{private => database}/genre/model.py | 0 data_scrapers/{private => database}/genre/repository.py | 0 data_scrapers/{private => database}/lambda_func.py | 0 data_scrapers/{private => database}/logger.py | 0 data_scrapers/{private => database}/model.py | 0 data_scrapers/{private => database}/screenshot/model.py | 0 data_scrapers/{private => database}/screenshot/repository.py | 0 data_scrapers/{private => database}/screenshot/service.py | 0 data_scrapers/{public => game_updater}/aws_lambda/event.py | 0 data_scrapers/{public => game_updater}/aws_lambda/exception.py | 0 data_scrapers/{public => game_updater}/aws_lambda/lambda_api.py | 0 data_scrapers/{public => game_updater}/aws_lambda/model.py | 0 data_scrapers/{public => game_updater}/config.py | 0 data_scrapers/{public => game_updater}/lambda_func.py | 0 data_scrapers/{public => game_updater}/logger.py | 0 data_scrapers/{public => game_updater}/protocols.py | 0 data_scrapers/{public => game_updater}/scraper/game.py | 0 data_scrapers/{public => game_updater}/scraper/model.py | 0 data_scrapers/{public => game_updater}/scraper/screenshot.py | 0 data_scrapers/{public => game_updater}/scraper/service.py | 0 data_scrapers/{public => game_updater}/steam/exception.py | 0 data_scrapers/{public => game_updater}/steam/gamalytic_api.py | 0 data_scrapers/{public => game_updater}/steam/model.py | 0 data_scrapers/{public => game_updater}/steam/steam_api.py | 0 data_scrapers/{public => game_updater}/steam/steampowered_api.py | 0 31 files changed, 0 insertions(+), 0 deletions(-) rename data_scrapers/{private => database}/config.py (100%) rename data_scrapers/{private => database}/database.py (100%) rename data_scrapers/{private => database}/event.py (100%) rename data_scrapers/{private => database}/game/model.py (100%) rename data_scrapers/{private => database}/game/repository.py (100%) rename data_scrapers/{private => database}/game/service.py (100%) rename data_scrapers/{private => database}/genre/model.py (100%) rename data_scrapers/{private => database}/genre/repository.py (100%) rename data_scrapers/{private => database}/lambda_func.py (100%) rename data_scrapers/{private => database}/logger.py (100%) rename data_scrapers/{private => database}/model.py (100%) rename data_scrapers/{private => database}/screenshot/model.py (100%) rename data_scrapers/{private => database}/screenshot/repository.py (100%) rename data_scrapers/{private => database}/screenshot/service.py (100%) rename data_scrapers/{public => game_updater}/aws_lambda/event.py (100%) rename data_scrapers/{public => game_updater}/aws_lambda/exception.py (100%) rename data_scrapers/{public => game_updater}/aws_lambda/lambda_api.py (100%) rename data_scrapers/{public => game_updater}/aws_lambda/model.py (100%) rename data_scrapers/{public => game_updater}/config.py (100%) rename data_scrapers/{public => game_updater}/lambda_func.py (100%) rename data_scrapers/{public => game_updater}/logger.py (100%) rename data_scrapers/{public => game_updater}/protocols.py (100%) rename data_scrapers/{public => game_updater}/scraper/game.py (100%) rename data_scrapers/{public => game_updater}/scraper/model.py (100%) rename data_scrapers/{public => game_updater}/scraper/screenshot.py (100%) rename data_scrapers/{public => game_updater}/scraper/service.py (100%) rename data_scrapers/{public => game_updater}/steam/exception.py (100%) rename data_scrapers/{public => game_updater}/steam/gamalytic_api.py (100%) rename data_scrapers/{public => game_updater}/steam/model.py (100%) rename data_scrapers/{public => game_updater}/steam/steam_api.py (100%) rename data_scrapers/{public => game_updater}/steam/steampowered_api.py (100%) diff --git a/data_scrapers/private/config.py b/data_scrapers/database/config.py similarity index 100% rename from data_scrapers/private/config.py rename to data_scrapers/database/config.py diff --git a/data_scrapers/private/database.py b/data_scrapers/database/database.py similarity index 100% rename from data_scrapers/private/database.py rename to data_scrapers/database/database.py diff --git a/data_scrapers/private/event.py b/data_scrapers/database/event.py similarity index 100% rename from data_scrapers/private/event.py rename to data_scrapers/database/event.py diff --git a/data_scrapers/private/game/model.py b/data_scrapers/database/game/model.py similarity index 100% rename from data_scrapers/private/game/model.py rename to data_scrapers/database/game/model.py diff --git a/data_scrapers/private/game/repository.py b/data_scrapers/database/game/repository.py similarity index 100% rename from data_scrapers/private/game/repository.py rename to data_scrapers/database/game/repository.py diff --git a/data_scrapers/private/game/service.py b/data_scrapers/database/game/service.py similarity index 100% rename from data_scrapers/private/game/service.py rename to data_scrapers/database/game/service.py diff --git a/data_scrapers/private/genre/model.py b/data_scrapers/database/genre/model.py similarity index 100% rename from data_scrapers/private/genre/model.py rename to data_scrapers/database/genre/model.py diff --git a/data_scrapers/private/genre/repository.py b/data_scrapers/database/genre/repository.py similarity index 100% rename from data_scrapers/private/genre/repository.py rename to data_scrapers/database/genre/repository.py diff --git a/data_scrapers/private/lambda_func.py b/data_scrapers/database/lambda_func.py similarity index 100% rename from data_scrapers/private/lambda_func.py rename to data_scrapers/database/lambda_func.py diff --git a/data_scrapers/private/logger.py b/data_scrapers/database/logger.py similarity index 100% rename from data_scrapers/private/logger.py rename to data_scrapers/database/logger.py diff --git a/data_scrapers/private/model.py b/data_scrapers/database/model.py similarity index 100% rename from data_scrapers/private/model.py rename to data_scrapers/database/model.py diff --git a/data_scrapers/private/screenshot/model.py b/data_scrapers/database/screenshot/model.py similarity index 100% rename from data_scrapers/private/screenshot/model.py rename to data_scrapers/database/screenshot/model.py diff --git a/data_scrapers/private/screenshot/repository.py b/data_scrapers/database/screenshot/repository.py similarity index 100% rename from data_scrapers/private/screenshot/repository.py rename to data_scrapers/database/screenshot/repository.py diff --git a/data_scrapers/private/screenshot/service.py b/data_scrapers/database/screenshot/service.py similarity index 100% rename from data_scrapers/private/screenshot/service.py rename to data_scrapers/database/screenshot/service.py diff --git a/data_scrapers/public/aws_lambda/event.py b/data_scrapers/game_updater/aws_lambda/event.py similarity index 100% rename from data_scrapers/public/aws_lambda/event.py rename to data_scrapers/game_updater/aws_lambda/event.py diff --git a/data_scrapers/public/aws_lambda/exception.py b/data_scrapers/game_updater/aws_lambda/exception.py similarity index 100% rename from data_scrapers/public/aws_lambda/exception.py rename to data_scrapers/game_updater/aws_lambda/exception.py diff --git a/data_scrapers/public/aws_lambda/lambda_api.py b/data_scrapers/game_updater/aws_lambda/lambda_api.py similarity index 100% rename from data_scrapers/public/aws_lambda/lambda_api.py rename to data_scrapers/game_updater/aws_lambda/lambda_api.py diff --git a/data_scrapers/public/aws_lambda/model.py b/data_scrapers/game_updater/aws_lambda/model.py similarity index 100% rename from data_scrapers/public/aws_lambda/model.py rename to data_scrapers/game_updater/aws_lambda/model.py diff --git a/data_scrapers/public/config.py b/data_scrapers/game_updater/config.py similarity index 100% rename from data_scrapers/public/config.py rename to data_scrapers/game_updater/config.py diff --git a/data_scrapers/public/lambda_func.py b/data_scrapers/game_updater/lambda_func.py similarity index 100% rename from data_scrapers/public/lambda_func.py rename to data_scrapers/game_updater/lambda_func.py diff --git a/data_scrapers/public/logger.py b/data_scrapers/game_updater/logger.py similarity index 100% rename from data_scrapers/public/logger.py rename to data_scrapers/game_updater/logger.py diff --git a/data_scrapers/public/protocols.py b/data_scrapers/game_updater/protocols.py similarity index 100% rename from data_scrapers/public/protocols.py rename to data_scrapers/game_updater/protocols.py diff --git a/data_scrapers/public/scraper/game.py b/data_scrapers/game_updater/scraper/game.py similarity index 100% rename from data_scrapers/public/scraper/game.py rename to data_scrapers/game_updater/scraper/game.py diff --git a/data_scrapers/public/scraper/model.py b/data_scrapers/game_updater/scraper/model.py similarity index 100% rename from data_scrapers/public/scraper/model.py rename to data_scrapers/game_updater/scraper/model.py diff --git a/data_scrapers/public/scraper/screenshot.py b/data_scrapers/game_updater/scraper/screenshot.py similarity index 100% rename from data_scrapers/public/scraper/screenshot.py rename to data_scrapers/game_updater/scraper/screenshot.py diff --git a/data_scrapers/public/scraper/service.py b/data_scrapers/game_updater/scraper/service.py similarity index 100% rename from data_scrapers/public/scraper/service.py rename to data_scrapers/game_updater/scraper/service.py diff --git a/data_scrapers/public/steam/exception.py b/data_scrapers/game_updater/steam/exception.py similarity index 100% rename from data_scrapers/public/steam/exception.py rename to data_scrapers/game_updater/steam/exception.py diff --git a/data_scrapers/public/steam/gamalytic_api.py b/data_scrapers/game_updater/steam/gamalytic_api.py similarity index 100% rename from data_scrapers/public/steam/gamalytic_api.py rename to data_scrapers/game_updater/steam/gamalytic_api.py diff --git a/data_scrapers/public/steam/model.py b/data_scrapers/game_updater/steam/model.py similarity index 100% rename from data_scrapers/public/steam/model.py rename to data_scrapers/game_updater/steam/model.py diff --git a/data_scrapers/public/steam/steam_api.py b/data_scrapers/game_updater/steam/steam_api.py similarity index 100% rename from data_scrapers/public/steam/steam_api.py rename to data_scrapers/game_updater/steam/steam_api.py diff --git a/data_scrapers/public/steam/steampowered_api.py b/data_scrapers/game_updater/steam/steampowered_api.py similarity index 100% rename from data_scrapers/public/steam/steampowered_api.py rename to data_scrapers/game_updater/steam/steampowered_api.py From 4c5400a77c2c068b201062277ffff6d178ea1373 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Sun, 4 Feb 2024 03:07:55 +0900 Subject: [PATCH 02/31] =?UTF-8?q?refacotr:=20=EB=9E=8C=EB=8B=A4=20?= =?UTF-8?q?=EB=94=94=EB=A0=89=ED=84=B0=EB=A6=AC=20=EC=9D=B4=EB=A6=84=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD=20=EB=B0=8F=20import=20=EB=AC=B8=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/__init__.py | 0 data_scrapers/{database => database_lambda}/config.py | 0 .../{database => database_lambda}/database.py | 0 data_scrapers/{database => database_lambda}/event.py | 0 .../{database => database_lambda}/game/model.py | 0 .../{database => database_lambda}/game/repository.py | 0 .../{database => database_lambda}/game/service.py | 0 .../{database => database_lambda}/genre/model.py | 0 .../{database => database_lambda}/genre/repository.py | 0 .../{database => database_lambda}/lambda_func.py | 10 +++++----- data_scrapers/{database => database_lambda}/logger.py | 0 data_scrapers/{database => database_lambda}/model.py | 0 .../{database => database_lambda}/screenshot/model.py | 0 .../screenshot/repository.py | 0 .../screenshot/service.py | 0 data_scrapers/game_updater/aws_lambda/lambda_api.py | 9 ++++----- data_scrapers/game_updater/config.py | 1 + data_scrapers/game_updater/lambda_func.py | 10 +++++----- ...ild_private_lambda.sh => build_database_lambda.sh} | 6 +++--- ..._public_lambda.sh => build_game_updater_lambda.sh} | 6 +++--- data_scrapers/scripts/reset_database.py | 11 +++++------ data_scrapers/scripts/save_games.py | 4 ++-- .../tests/{private => database_lambda}/conftest.py | 2 +- .../tests/{private => database_lambda}/database.py | 10 +++++----- .../game/test_game_service.py | 8 ++++---- .../screenshot/test_screenshot_service.py | 8 ++++---- .../tests/{private => database_lambda}/utils/game.py | 4 ++-- .../tests/{private => database_lambda}/utils/genre.py | 2 +- .../{private => database_lambda}/utils/screenshot.py | 2 +- .../tests/{private => database_lambda}/utils/utils.py | 0 .../scraper/test_game_scraper.py | 10 +++++----- .../scraper/test_screenshot_scraper.py | 8 ++++---- .../{public => game_updater}/steam/test_steam_api.py | 4 ++-- .../{public => game_updater}/utils/mock_lambda_api.py | 4 ++-- .../{public => game_updater}/utils/mock_steam_api.py | 6 +++--- .../tests/{public => game_updater}/utils/model.py | 2 +- .../tests/{public => game_updater}/utils/steam.py | 0 .../tests/{public => game_updater}/utils/utils.py | 0 38 files changed, 63 insertions(+), 64 deletions(-) create mode 100644 data_scrapers/database_lambda/__init__.py rename data_scrapers/{database => database_lambda}/config.py (100%) rename data_scrapers/{database => database_lambda}/database.py (100%) rename data_scrapers/{database => database_lambda}/event.py (100%) rename data_scrapers/{database => database_lambda}/game/model.py (100%) rename data_scrapers/{database => database_lambda}/game/repository.py (100%) rename data_scrapers/{database => database_lambda}/game/service.py (100%) rename data_scrapers/{database => database_lambda}/genre/model.py (100%) rename data_scrapers/{database => database_lambda}/genre/repository.py (100%) rename data_scrapers/{database => database_lambda}/lambda_func.py (71%) rename data_scrapers/{database => database_lambda}/logger.py (100%) rename data_scrapers/{database => database_lambda}/model.py (100%) rename data_scrapers/{database => database_lambda}/screenshot/model.py (100%) rename data_scrapers/{database => database_lambda}/screenshot/repository.py (100%) rename data_scrapers/{database => database_lambda}/screenshot/service.py (100%) rename data_scrapers/scripts/{build_private_lambda.sh => build_database_lambda.sh} (74%) rename data_scrapers/scripts/{build_public_lambda.sh => build_game_updater_lambda.sh} (73%) rename data_scrapers/tests/{private => database_lambda}/conftest.py (79%) rename data_scrapers/tests/{private => database_lambda}/database.py (69%) rename data_scrapers/tests/{private => database_lambda}/game/test_game_service.py (91%) rename data_scrapers/tests/{private => database_lambda}/screenshot/test_screenshot_service.py (78%) rename data_scrapers/tests/{private => database_lambda}/utils/game.py (91%) rename data_scrapers/tests/{private => database_lambda}/utils/genre.py (90%) rename data_scrapers/tests/{private => database_lambda}/utils/screenshot.py (91%) rename data_scrapers/tests/{private => database_lambda}/utils/utils.py (100%) rename data_scrapers/tests/{public => game_updater}/scraper/test_game_scraper.py (82%) rename data_scrapers/tests/{public => game_updater}/scraper/test_screenshot_scraper.py (82%) rename data_scrapers/tests/{public => game_updater}/steam/test_steam_api.py (89%) rename data_scrapers/tests/{public => game_updater}/utils/mock_lambda_api.py (91%) rename data_scrapers/tests/{public => game_updater}/utils/mock_steam_api.py (95%) rename data_scrapers/tests/{public => game_updater}/utils/model.py (95%) rename data_scrapers/tests/{public => game_updater}/utils/steam.py (100%) rename data_scrapers/tests/{public => game_updater}/utils/utils.py (100%) diff --git a/data_scrapers/database_lambda/__init__.py b/data_scrapers/database_lambda/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_scrapers/database/config.py b/data_scrapers/database_lambda/config.py similarity index 100% rename from data_scrapers/database/config.py rename to data_scrapers/database_lambda/config.py diff --git a/data_scrapers/database/database.py b/data_scrapers/database_lambda/database.py similarity index 100% rename from data_scrapers/database/database.py rename to data_scrapers/database_lambda/database.py diff --git a/data_scrapers/database/event.py b/data_scrapers/database_lambda/event.py similarity index 100% rename from data_scrapers/database/event.py rename to data_scrapers/database_lambda/event.py diff --git a/data_scrapers/database/game/model.py b/data_scrapers/database_lambda/game/model.py similarity index 100% rename from data_scrapers/database/game/model.py rename to data_scrapers/database_lambda/game/model.py diff --git a/data_scrapers/database/game/repository.py b/data_scrapers/database_lambda/game/repository.py similarity index 100% rename from data_scrapers/database/game/repository.py rename to data_scrapers/database_lambda/game/repository.py diff --git a/data_scrapers/database/game/service.py b/data_scrapers/database_lambda/game/service.py similarity index 100% rename from data_scrapers/database/game/service.py rename to data_scrapers/database_lambda/game/service.py diff --git a/data_scrapers/database/genre/model.py b/data_scrapers/database_lambda/genre/model.py similarity index 100% rename from data_scrapers/database/genre/model.py rename to data_scrapers/database_lambda/genre/model.py diff --git a/data_scrapers/database/genre/repository.py b/data_scrapers/database_lambda/genre/repository.py similarity index 100% rename from data_scrapers/database/genre/repository.py rename to data_scrapers/database_lambda/genre/repository.py diff --git a/data_scrapers/database/lambda_func.py b/data_scrapers/database_lambda/lambda_func.py similarity index 71% rename from data_scrapers/database/lambda_func.py rename to data_scrapers/database_lambda/lambda_func.py index 4de4876..560a5ff 100644 --- a/data_scrapers/database/lambda_func.py +++ b/data_scrapers/database_lambda/lambda_func.py @@ -2,11 +2,11 @@ from sqlalchemy.orm import Session -from private.database import engine, init_database -from private.event import Event, EventName -from private.game.service import get_games_in_steam_ids, get_some_games, save_games -from private.logger import logger -from private.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots +from database_lambda.database import engine, init_database +from database_lambda.event import Event, EventName +from database_lambda.game.service import get_games_in_steam_ids, get_some_games, save_games +from database_lambda.logger import logger +from database_lambda.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots funcs: dict[EventName, Callable[..., Any]] = { "save_games": save_games, diff --git a/data_scrapers/database/logger.py b/data_scrapers/database_lambda/logger.py similarity index 100% rename from data_scrapers/database/logger.py rename to data_scrapers/database_lambda/logger.py diff --git a/data_scrapers/database/model.py b/data_scrapers/database_lambda/model.py similarity index 100% rename from data_scrapers/database/model.py rename to data_scrapers/database_lambda/model.py diff --git a/data_scrapers/database/screenshot/model.py b/data_scrapers/database_lambda/screenshot/model.py similarity index 100% rename from data_scrapers/database/screenshot/model.py rename to data_scrapers/database_lambda/screenshot/model.py diff --git a/data_scrapers/database/screenshot/repository.py b/data_scrapers/database_lambda/screenshot/repository.py similarity index 100% rename from data_scrapers/database/screenshot/repository.py rename to data_scrapers/database_lambda/screenshot/repository.py diff --git a/data_scrapers/database/screenshot/service.py b/data_scrapers/database_lambda/screenshot/service.py similarity index 100% rename from data_scrapers/database/screenshot/service.py rename to data_scrapers/database_lambda/screenshot/service.py diff --git a/data_scrapers/game_updater/aws_lambda/lambda_api.py b/data_scrapers/game_updater/aws_lambda/lambda_api.py index af109b7..bac0522 100644 --- a/data_scrapers/game_updater/aws_lambda/lambda_api.py +++ b/data_scrapers/game_updater/aws_lambda/lambda_api.py @@ -3,21 +3,20 @@ import boto3 -from public.aws_lambda.exception import AWSLambdaException - from .. import protocols +from ..config import setting from ..scraper.model import NewGameScreenshot from .event import Event +from .exception import AWSLambdaException from .model import Game, GameScreenshot, SaveGame class LambdaAPI(protocols.LambdaAPI): - def __init__(self, private_function_name: str) -> None: - self.private_function_name = private_function_name + def __init__(self) -> None: self.client = boto3.client("lambda") def invoke_lambda(self, event: Event) -> Any: - response = self.client.invoke(FunctionName=self.private_function_name, Payload=json.dumps(event)) + response = self.client.invoke(FunctionName=setting.DATABASE_LAMBDA_NAME, Payload=json.dumps(event)) if "FunctionError" in response: raise AWSLambdaException(response["FunctionError"]) diff --git a/data_scrapers/game_updater/config.py b/data_scrapers/game_updater/config.py index 28be26d..e4a09aa 100644 --- a/data_scrapers/game_updater/config.py +++ b/data_scrapers/game_updater/config.py @@ -4,6 +4,7 @@ class Config(BaseSettings): WORKER_CNT: int = 10 MIN_REVENUE: int = 10000000 # 10M + DATABASE_LAMBDA_NAME: str = "database" model_config = SettingsConfigDict(env_file=".scraper.env", env_file_encoding="utf-8") diff --git a/data_scrapers/game_updater/lambda_func.py b/data_scrapers/game_updater/lambda_func.py index ecf4158..f67cf02 100644 --- a/data_scrapers/game_updater/lambda_func.py +++ b/data_scrapers/game_updater/lambda_func.py @@ -1,9 +1,9 @@ from typing import Any -from public.aws_lambda.lambda_api import LambdaAPI -from public.logger import logger -from public.scraper.service import scrap_game_screenshot_for_all, scrap_games -from public.steam.steam_api import SteamAPI +from game_updater.aws_lambda.lambda_api import LambdaAPI +from game_updater.logger import logger +from game_updater.scraper.service import scrap_game_screenshot_for_all, scrap_games +from game_updater.steam.steam_api import SteamAPI def scrap_games_job(lambda_api: LambdaAPI): @@ -23,7 +23,7 @@ def scrap_screenshots_job(lambda_api: LambdaAPI): def lambda_handler(event: Any, context: Any): - lambda_api = LambdaAPI("private") + lambda_api = LambdaAPI() scrap_games_job(lambda_api) # scrap_screenshots_job(lambda_api) diff --git a/data_scrapers/scripts/build_private_lambda.sh b/data_scrapers/scripts/build_database_lambda.sh similarity index 74% rename from data_scrapers/scripts/build_private_lambda.sh rename to data_scrapers/scripts/build_database_lambda.sh index c8cc2c5..9f4e5eb 100644 --- a/data_scrapers/scripts/build_private_lambda.sh +++ b/data_scrapers/scripts/build_database_lambda.sh @@ -1,13 +1,13 @@ mkdir ./build # remove previous build -rm ./build/private_lambda.zip +rm ./build/database_lambda.zip # get dependency poetry export -f requirements.txt --without-hashes > requirements.txt pip install --platform manylinux2014_x86_64 --implementation cp --python-version 3.9 --only-binary=:all: --upgrade -r requirements.txt -t ./package # zip package with source codes cd ./package -zip -r ../build/private_lambda.zip * +zip -r ../build/database_lambda.zip * cd .. -zip -r ./build/private_lambda.zip private +zip -r ./build/database_lambda.zip database_lambda # remove packages rm -r ./package \ No newline at end of file diff --git a/data_scrapers/scripts/build_public_lambda.sh b/data_scrapers/scripts/build_game_updater_lambda.sh similarity index 73% rename from data_scrapers/scripts/build_public_lambda.sh rename to data_scrapers/scripts/build_game_updater_lambda.sh index 40d5b93..7e934db 100644 --- a/data_scrapers/scripts/build_public_lambda.sh +++ b/data_scrapers/scripts/build_game_updater_lambda.sh @@ -1,13 +1,13 @@ mkdir ./build # remove previous build -rm ./build/public_lambda.zip +rm ./build/game_updater_lambda.zip # get dependency poetry export -f requirements.txt --without-hashes > requirements.txt pip install --platform manylinux2014_x86_64 --implementation cp --python-version 3.9 --only-binary=:all: --upgrade -r requirements.txt -t ./package # zip package with source codes cd ./package -zip -r ../build/public_lambda.zip * +zip -r ../build/game_updater_lambda.zip * cd .. -zip -r ./build/public_lambda.zip public +zip -r ./build/game_updater_lambda.zip game_updater # remove packages rm -r ./package \ No newline at end of file diff --git a/data_scrapers/scripts/reset_database.py b/data_scrapers/scripts/reset_database.py index a7a396e..19f6401 100644 --- a/data_scrapers/scripts/reset_database.py +++ b/data_scrapers/scripts/reset_database.py @@ -5,11 +5,10 @@ sys.path.append(str(pathlib.Path(__file__).resolve().parents[1])) -import private.genre.model # noqa: E402, F401 -import private.screenshot.model # noqa: E402, F401 -from private.config import setting # noqa: E402 -from private.database import Base, engine # noqa: E402 -from private.game.model import game_genre_link # noqa: E402 +import database_lambda.genre.model # noqa: E402, F401 +import database_lambda.screenshot.model # noqa: E402, F401 +from database_lambda.config import setting # noqa: E402 +from database_lambda.database import Base, engine # noqa: E402 if database_exists(setting.DATABASE_URL): drop_database(setting.DATABASE_URL) @@ -17,5 +16,5 @@ create_database(setting.DATABASE_URL) -tables = list(Base.metadata.tables.values()) + [game_genre_link] +tables = list(Base.metadata.tables.values()) Base.metadata.create_all(engine, tables, checkfirst=True) diff --git a/data_scrapers/scripts/save_games.py b/data_scrapers/scripts/save_games.py index 94732ce..2a9e756 100644 --- a/data_scrapers/scripts/save_games.py +++ b/data_scrapers/scripts/save_games.py @@ -8,8 +8,8 @@ sys.path.append(str(pathlib.Path(__file__).resolve().parents[1])) -from private.database import engine # noqa: E402 -from private.game import service # noqa: E402 +from database_lambda.database import engine # noqa: E402 +from database_lambda.game import service # noqa: E402 class Game(TypedDict): diff --git a/data_scrapers/tests/private/conftest.py b/data_scrapers/tests/database_lambda/conftest.py similarity index 79% rename from data_scrapers/tests/private/conftest.py rename to data_scrapers/tests/database_lambda/conftest.py index 9985e49..e190a35 100644 --- a/data_scrapers/tests/private/conftest.py +++ b/data_scrapers/tests/database_lambda/conftest.py @@ -3,7 +3,7 @@ import pytest from sqlalchemy.orm import Session -from tests.private.database import create_tables, drop_tables, engine, init_database +from tests.database_lambda.database import create_tables, drop_tables, engine, init_database @pytest.fixture(autouse=True) diff --git a/data_scrapers/tests/private/database.py b/data_scrapers/tests/database_lambda/database.py similarity index 69% rename from data_scrapers/tests/private/database.py rename to data_scrapers/tests/database_lambda/database.py index e2b6c41..1c1e999 100644 --- a/data_scrapers/tests/private/database.py +++ b/data_scrapers/tests/database_lambda/database.py @@ -1,11 +1,11 @@ from sqlalchemy import Table, create_engine from sqlalchemy_utils import create_database, database_exists -import private.game.model # noqa: F401 -import private.genre.model # noqa: F401 -import private.screenshot.model # noqa: F401 -from private.config import setting -from private.model import Base +import database_lambda.game.model # noqa: F401 +import database_lambda.genre.model # noqa: F401 +import database_lambda.screenshot.model # noqa: F401 +from database_lambda.config import setting +from database_lambda.model import Base engine = create_engine(setting.DATABASE_URL, echo=True) # type: ignore diff --git a/data_scrapers/tests/private/game/test_game_service.py b/data_scrapers/tests/database_lambda/game/test_game_service.py similarity index 91% rename from data_scrapers/tests/private/game/test_game_service.py rename to data_scrapers/tests/database_lambda/game/test_game_service.py index afd1d34..e482ea8 100644 --- a/data_scrapers/tests/private/game/test_game_service.py +++ b/data_scrapers/tests/database_lambda/game/test_game_service.py @@ -1,10 +1,10 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from private.game.model import Game -from private.game.service import get_games_in_steam_ids, get_some_games, save_games -from tests.private.utils.game import create_random_game -from tests.private.utils.utils import random_datetime +from database_lambda.game.model import Game +from database_lambda.game.service import get_games_in_steam_ids, get_some_games, save_games +from tests.database_lambda.utils.game import create_random_game +from tests.database_lambda.utils.utils import random_datetime def test_get_some_games은_게임을_가져와야한다(session: Session): diff --git a/data_scrapers/tests/private/screenshot/test_screenshot_service.py b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py similarity index 78% rename from data_scrapers/tests/private/screenshot/test_screenshot_service.py rename to data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py index b918689..b52b735 100644 --- a/data_scrapers/tests/private/screenshot/test_screenshot_service.py +++ b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py @@ -1,10 +1,10 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from private.screenshot.model import GameScreenshot -from private.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots -from tests.private.utils.game import create_random_game -from tests.private.utils.screenshot import create_random_game_screenshot +from database_lambda.screenshot.model import GameScreenshot +from database_lambda.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots +from tests.database_lambda.utils.game import create_random_game +from tests.database_lambda.utils.screenshot import create_random_game_screenshot def test_save_screenshots은_입력한_스크린샷을_저장해야_한다(session: Session): diff --git a/data_scrapers/tests/private/utils/game.py b/data_scrapers/tests/database_lambda/utils/game.py similarity index 91% rename from data_scrapers/tests/private/utils/game.py rename to data_scrapers/tests/database_lambda/utils/game.py index 69eb1af..933e057 100644 --- a/data_scrapers/tests/private/utils/game.py +++ b/data_scrapers/tests/database_lambda/utils/game.py @@ -4,8 +4,8 @@ from sqlalchemy.orm import Session -from private.game.model import Game -from private.genre.model import Genre +from database_lambda.game.model import Game +from database_lambda.genre.model import Genre from .genre import create_random_genre from .utils import random_datetime diff --git a/data_scrapers/tests/private/utils/genre.py b/data_scrapers/tests/database_lambda/utils/genre.py similarity index 90% rename from data_scrapers/tests/private/utils/genre.py rename to data_scrapers/tests/database_lambda/utils/genre.py index 87ffa24..55aa023 100644 --- a/data_scrapers/tests/private/utils/genre.py +++ b/data_scrapers/tests/database_lambda/utils/genre.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Session -from private.genre.model import Genre +from database_lambda.genre.model import Genre genre_counter = 1 diff --git a/data_scrapers/tests/private/utils/screenshot.py b/data_scrapers/tests/database_lambda/utils/screenshot.py similarity index 91% rename from data_scrapers/tests/private/utils/screenshot.py rename to data_scrapers/tests/database_lambda/utils/screenshot.py index a09a235..ff48003 100644 --- a/data_scrapers/tests/private/utils/screenshot.py +++ b/data_scrapers/tests/database_lambda/utils/screenshot.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Session -from private.screenshot.model import GameScreenshot +from database_lambda.screenshot.model import GameScreenshot from .game import create_random_game diff --git a/data_scrapers/tests/private/utils/utils.py b/data_scrapers/tests/database_lambda/utils/utils.py similarity index 100% rename from data_scrapers/tests/private/utils/utils.py rename to data_scrapers/tests/database_lambda/utils/utils.py diff --git a/data_scrapers/tests/public/scraper/test_game_scraper.py b/data_scrapers/tests/game_updater/scraper/test_game_scraper.py similarity index 82% rename from data_scrapers/tests/public/scraper/test_game_scraper.py rename to data_scrapers/tests/game_updater/scraper/test_game_scraper.py index 140a8e9..078ff0f 100644 --- a/data_scrapers/tests/public/scraper/test_game_scraper.py +++ b/data_scrapers/tests/game_updater/scraper/test_game_scraper.py @@ -1,10 +1,10 @@ import pytest -from public.config import setting -from public.scraper.game import scrap_games -from tests.public.utils.mock_lambda_api import MockLambdaAPI -from tests.public.utils.mock_steam_api import MockSteamAPI -from tests.public.utils.steam import create_random_game +from game_updater.config import setting +from game_updater.scraper.game import scrap_games +from tests.game_updater.utils.mock_lambda_api import MockLambdaAPI +from tests.game_updater.utils.mock_steam_api import MockSteamAPI +from tests.game_updater.utils.steam import create_random_game @pytest.fixture diff --git a/data_scrapers/tests/public/scraper/test_screenshot_scraper.py b/data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py similarity index 82% rename from data_scrapers/tests/public/scraper/test_screenshot_scraper.py rename to data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py index eedc1b2..1322d0d 100644 --- a/data_scrapers/tests/public/scraper/test_screenshot_scraper.py +++ b/data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py @@ -1,9 +1,9 @@ import pytest -from public.scraper.screenshot import scrap_game_screenshot -from tests.public.utils.mock_lambda_api import MockLambdaAPI -from tests.public.utils.mock_steam_api import MockSteamAPI -from tests.public.utils.model import create_random_game +from game_updater.scraper.screenshot import scrap_game_screenshot +from tests.game_updater.utils.mock_lambda_api import MockLambdaAPI +from tests.game_updater.utils.mock_steam_api import MockSteamAPI +from tests.game_updater.utils.model import create_random_game @pytest.fixture diff --git a/data_scrapers/tests/public/steam/test_steam_api.py b/data_scrapers/tests/game_updater/steam/test_steam_api.py similarity index 89% rename from data_scrapers/tests/public/steam/test_steam_api.py rename to data_scrapers/tests/game_updater/steam/test_steam_api.py index cae5e27..7e50bb8 100644 --- a/data_scrapers/tests/public/steam/test_steam_api.py +++ b/data_scrapers/tests/game_updater/steam/test_steam_api.py @@ -1,7 +1,7 @@ import pytest -from public.steam.exception import SteamAPINoContentsException -from public.steam.steam_api import SteamAPI +from game_updater.steam.exception import SteamAPINoContentsException +from game_updater.steam.steam_api import SteamAPI @pytest.fixture diff --git a/data_scrapers/tests/public/utils/mock_lambda_api.py b/data_scrapers/tests/game_updater/utils/mock_lambda_api.py similarity index 91% rename from data_scrapers/tests/public/utils/mock_lambda_api.py rename to data_scrapers/tests/game_updater/utils/mock_lambda_api.py index e79c8f8..81ef076 100644 --- a/data_scrapers/tests/public/utils/mock_lambda_api.py +++ b/data_scrapers/tests/game_updater/utils/mock_lambda_api.py @@ -1,8 +1,8 @@ from datetime import datetime from typing import Sequence -from public.aws_lambda.model import Game, GameScreenshot, SaveGame -from public.protocols import LambdaAPI +from game_updater.aws_lambda.model import Game, GameScreenshot, SaveGame +from game_updater.protocols import LambdaAPI class MockLambdaAPI(LambdaAPI): diff --git a/data_scrapers/tests/public/utils/mock_steam_api.py b/data_scrapers/tests/game_updater/utils/mock_steam_api.py similarity index 95% rename from data_scrapers/tests/public/utils/mock_steam_api.py rename to data_scrapers/tests/game_updater/utils/mock_steam_api.py index f69628c..4ac57e2 100644 --- a/data_scrapers/tests/public/utils/mock_steam_api.py +++ b/data_scrapers/tests/game_updater/utils/mock_steam_api.py @@ -1,9 +1,9 @@ from random import choice, randint, sample from typing import Optional -from public.protocols import SteamAPI -from public.steam.exception import SteamAPINoContentsException -from public.steam.model import ( +from game_updater.protocols import SteamAPI +from game_updater.steam.exception import SteamAPINoContentsException +from game_updater.steam.model import ( GamalyticSteamGameDetailResponse, GamalyticSteamGameResponse, SteamFeatureGameResponse, diff --git a/data_scrapers/tests/public/utils/model.py b/data_scrapers/tests/game_updater/utils/model.py similarity index 95% rename from data_scrapers/tests/public/utils/model.py rename to data_scrapers/tests/game_updater/utils/model.py index 4806ce3..a7fb686 100644 --- a/data_scrapers/tests/public/utils/model.py +++ b/data_scrapers/tests/game_updater/utils/model.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional -from public.aws_lambda.model import Game +from game_updater.aws_lambda.model import Game from .utils import random_datetime diff --git a/data_scrapers/tests/public/utils/steam.py b/data_scrapers/tests/game_updater/utils/steam.py similarity index 100% rename from data_scrapers/tests/public/utils/steam.py rename to data_scrapers/tests/game_updater/utils/steam.py diff --git a/data_scrapers/tests/public/utils/utils.py b/data_scrapers/tests/game_updater/utils/utils.py similarity index 100% rename from data_scrapers/tests/public/utils/utils.py rename to data_scrapers/tests/game_updater/utils/utils.py From 648a9098bc48169ef5b6e5c9de3415c364f81201 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Mon, 5 Feb 2024 02:09:32 +0900 Subject: [PATCH 03/31] =?UTF-8?q?feat:=20=EB=8D=B0=EC=9D=BC=EB=A6=AC=20?= =?UTF-8?q?=ED=80=B4=EC=A6=88=20=EC=83=9D=EC=84=B1=EA=B8=B0=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/aws_lambda/event.py | 8 ++ .../daily_quiz/aws_lambda/exception.py | 2 + .../daily_quiz/aws_lambda/lambda_api.py | 38 ++++++++++ data_scrapers/daily_quiz/aws_lambda/model.py | 25 +++++++ data_scrapers/daily_quiz/config.py | 24 ++++++ .../daily_quiz/daily_quiz/game_picker.py | 73 +++++++++++++++++++ .../daily_quiz/daily_quiz/genre_picker.py | 7 ++ .../daily_quiz/screenshot_scraper.py | 8 ++ .../daily_quiz/daily_quiz/serivce.py | 34 +++++++++ data_scrapers/daily_quiz/daily_quiz/utils.py | 16 ++++ data_scrapers/daily_quiz/lambda_func.py | 22 ++++++ data_scrapers/daily_quiz/logger.py | 4 + data_scrapers/daily_quiz/protocols.py | 20 +++++ data_scrapers/daily_quiz/steam/model.py | 6 ++ data_scrapers/daily_quiz/steam/steam_api.py | 15 ++++ .../daily_quiz/steam/steampowered_api.py | 46 ++++++++++++ 16 files changed, 348 insertions(+) create mode 100644 data_scrapers/daily_quiz/aws_lambda/event.py create mode 100644 data_scrapers/daily_quiz/aws_lambda/exception.py create mode 100644 data_scrapers/daily_quiz/aws_lambda/lambda_api.py create mode 100644 data_scrapers/daily_quiz/aws_lambda/model.py create mode 100644 data_scrapers/daily_quiz/config.py create mode 100644 data_scrapers/daily_quiz/daily_quiz/game_picker.py create mode 100644 data_scrapers/daily_quiz/daily_quiz/genre_picker.py create mode 100644 data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py create mode 100644 data_scrapers/daily_quiz/daily_quiz/serivce.py create mode 100644 data_scrapers/daily_quiz/daily_quiz/utils.py create mode 100644 data_scrapers/daily_quiz/lambda_func.py create mode 100644 data_scrapers/daily_quiz/logger.py create mode 100644 data_scrapers/daily_quiz/protocols.py create mode 100644 data_scrapers/daily_quiz/steam/model.py create mode 100644 data_scrapers/daily_quiz/steam/steam_api.py create mode 100644 data_scrapers/daily_quiz/steam/steampowered_api.py diff --git a/data_scrapers/daily_quiz/aws_lambda/event.py b/data_scrapers/daily_quiz/aws_lambda/event.py new file mode 100644 index 0000000..2ad4408 --- /dev/null +++ b/data_scrapers/daily_quiz/aws_lambda/event.py @@ -0,0 +1,8 @@ +from typing import Any, Literal, TypedDict + +EventName = Literal["save_screenshots", "save_quizzes", "get_all_games"] + + +class Event(TypedDict): + name: EventName + payload: Any diff --git a/data_scrapers/daily_quiz/aws_lambda/exception.py b/data_scrapers/daily_quiz/aws_lambda/exception.py new file mode 100644 index 0000000..57db591 --- /dev/null +++ b/data_scrapers/daily_quiz/aws_lambda/exception.py @@ -0,0 +1,2 @@ +class AWSLambdaException(Exception): + ... diff --git a/data_scrapers/daily_quiz/aws_lambda/lambda_api.py b/data_scrapers/daily_quiz/aws_lambda/lambda_api.py new file mode 100644 index 0000000..b4aec2a --- /dev/null +++ b/data_scrapers/daily_quiz/aws_lambda/lambda_api.py @@ -0,0 +1,38 @@ +import json +from typing import Any, Iterable + +import boto3 + +from daily_quiz.aws_lambda.model import Game + +from .. import protocols +from ..config import setting +from .event import Event +from .exception import AWSLambdaException +from .model import SaveGameScreenshot, SaveQuiz + + +class LambdaAPI(protocols.LambdaAPI): + def __init__(self) -> None: + self.client = boto3.client("lambda") + + def invoke_lambda(self, event: Event) -> Any: + response = self.client.invoke(FunctionName=setting.DATABASE_LAMBDA_NAME, Payload=json.dumps(event)) + + if "FunctionError" in response: + raise AWSLambdaException(response["FunctionError"]) + + payload: Any = response["Payload"].read().decode("utf-8") + return json.loads(payload) + + def save_screenshots(self, screenshots: Iterable[SaveGameScreenshot]): + event = Event(name="save_screenshots", payload=[s.model_dump() for s in screenshots]) + self.invoke_lambda(event) + + def save_quizzes(self, quizzes: Iterable[SaveQuiz]): + event = Event(name="save_quizzes", payload=[q.model_dump() for q in quizzes]) + self.invoke_lambda(event) + + def get_all_games(self) -> list[Game]: + event = Event(name="get_all_games", payload=None) + return self.invoke_lambda(event) diff --git a/data_scrapers/daily_quiz/aws_lambda/model.py b/data_scrapers/daily_quiz/aws_lambda/model.py new file mode 100644 index 0000000..aaed741 --- /dev/null +++ b/data_scrapers/daily_quiz/aws_lambda/model.py @@ -0,0 +1,25 @@ +from datetime import datetime +from typing import Optional, Sequence + +from pydantic import BaseModel + + +class Game(BaseModel): + id: int + steam_id: int + name: str + kr_name: Optional[str] + released_at: datetime + genres: Sequence[str] + updated_at: datetime + created_at: datetime + + +class SaveGameScreenshot(BaseModel): + steam_file_id: int + url: str + game_id: int + + +class SaveQuiz(BaseModel): + screenshots: Sequence[SaveGameScreenshot] diff --git a/data_scrapers/daily_quiz/config.py b/data_scrapers/daily_quiz/config.py new file mode 100644 index 0000000..24e4e8b --- /dev/null +++ b/data_scrapers/daily_quiz/config.py @@ -0,0 +1,24 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Config(BaseSettings): + DATABASE_LAMBDA_NAME: str = "database" + DAILY_QUIZ_CNT: int = 5 + GAME_GENERES: list[str] = [ + "Action", + "Adventure", + "Massively Multiplayer", + "Strategy", + "RPG", + "Indie", + "Simulation", + "Casual", + "Racing", + "Sports", + ] + OLDER_GAME_COUNT: int = 2 # Newer game count will be `DAILY_QUIZ_CNT` - `OLDER_GAME_COUNT` + + model_config = SettingsConfigDict(env_file=".daily_quiz.env", env_file_encoding="utf-8") + + +setting = Config() # type: ignore diff --git a/data_scrapers/daily_quiz/daily_quiz/game_picker.py b/data_scrapers/daily_quiz/daily_quiz/game_picker.py new file mode 100644 index 0000000..6ba6b3d --- /dev/null +++ b/data_scrapers/daily_quiz/daily_quiz/game_picker.py @@ -0,0 +1,73 @@ +import random +from collections import defaultdict +from datetime import datetime +from typing import Iterable, Sequence + +from ..aws_lambda.model import Game +from ..config import setting +from .utils import divide_randomly + +GameGroup = list[Game] + + +def _categorize_games_by_genre(games: Iterable[Game], genres: Iterable[str]) -> list[GameGroup]: + categorized: dict[str, GameGroup] = defaultdict(list) + + for game in games: + for genre in game.genres: + if genre not in genres: + continue + + categorized[genre].append(game) + + return list(categorized.values()) + + +def _filter_older_games(games: Iterable[Game], threshold_released_at: datetime) -> list[Game]: + return [game for game in games if game.released_at <= threshold_released_at] + + +def _filter_newer_games(games: Iterable[Game], threshold_released_at: datetime) -> list[Game]: + return [game for game in games if game.released_at > threshold_released_at] + + +def _get_median_released_at(games: Iterable[Game]) -> datetime: + released_ats = [game.released_at for game in games] + released_ats.sort() + return released_ats[len(released_ats) // 2] + + +def _pick_older_newer_games( + categorized_games: Sequence[GameGroup], median_released_at: datetime +) -> tuple[list[GameGroup], list[GameGroup]]: + older_part, newer_part = divide_randomly(categorized_games, setting.OLDER_GAME_COUNT) + + olders = [_filter_older_games(games, median_released_at) for games in older_part] + newers = [_filter_newer_games(games, median_released_at) for games in older_part] + + return olders, newers + + +def _pick_unique_per_category(categorized_games: Iterable[GameGroup]) -> set[Game]: + unique_games: set[Game] = set() + + for games in categorized_games: + games_ = list(set(games) - unique_games) + game = random.choice(games_) + unique_games.add(game) + + return unique_games + + +def pick_games( + games: Iterable[Game], + genres: Iterable[str], +): + categorized_games = _categorize_games_by_genre(games, genres) + + # 오래된 게임 / 최신 게임으로 분리 + median_released_at = _get_median_released_at(games) + olders, newers = _pick_older_newer_games(categorized_games, median_released_at) + + # 최종 게임 선발 + return _pick_unique_per_category(categorized_games) diff --git a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py new file mode 100644 index 0000000..1c5f0fd --- /dev/null +++ b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py @@ -0,0 +1,7 @@ +import random + +from ..config import setting + + +def pick_genres() -> list[str]: + return random.choices(setting.GAME_GENERES, k=5) diff --git a/data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py b/data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py new file mode 100644 index 0000000..cfa34b6 --- /dev/null +++ b/data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py @@ -0,0 +1,8 @@ +from ..aws_lambda.model import Game, SaveGameScreenshot +from ..protocols import SteamAPI + + +def scrap_screenshots(steam_api: SteamAPI, game: Game): + scraped = steam_api.get_game_screenshots(game.steam_id) + + return [SaveGameScreenshot(steam_file_id=s.file_id, url=s.full_image_url, game_id=game.id) for s in scraped] diff --git a/data_scrapers/daily_quiz/daily_quiz/serivce.py b/data_scrapers/daily_quiz/daily_quiz/serivce.py new file mode 100644 index 0000000..4bab0c8 --- /dev/null +++ b/data_scrapers/daily_quiz/daily_quiz/serivce.py @@ -0,0 +1,34 @@ +from typing import Iterable + +from ..aws_lambda.model import Game, SaveQuiz +from ..protocols import LambdaAPI, SteamAPI +from .game_picker import pick_games +from .genre_picker import pick_genres +from .screenshot_scraper import scrap_screenshots + + +def create_quizzes(steam_api: SteamAPI, games: Iterable[Game]) -> list[SaveQuiz]: + quizzes = [] + for game in games: + # 스크린샷 크롤링 + screenshots = scrap_screenshots(steam_api, game) + quizzes.append(SaveQuiz(screenshots=screenshots)) + + return quizzes + + +def new_daily_quizzes(lambda_api: LambdaAPI, steam_api: SteamAPI): + # 모든 게임 가져오기 + all_games = lambda_api.get_all_games() + + # 장르 선택 + picked_genres = pick_genres() + + # 게임 선택 + picked_games = pick_games(all_games, picked_genres) + + # 퀴즈 생성 (스크린샷 스크래핑함) + quizzes = create_quizzes(steam_api, picked_games) + + # 저장 + lambda_api.save_quizzes(quizzes) diff --git a/data_scrapers/daily_quiz/daily_quiz/utils.py b/data_scrapers/daily_quiz/daily_quiz/utils.py new file mode 100644 index 0000000..4436964 --- /dev/null +++ b/data_scrapers/daily_quiz/daily_quiz/utils.py @@ -0,0 +1,16 @@ +import random +from typing import Sequence, TypeVar + +T = TypeVar("T") + + +def divide_randomly(x: Sequence[T], k: int) -> tuple[list[T], list[T]]: + idxes = range(len(x)) + + a_idxes = random.choices(idxes, k=k) + b_idxes = list(set(idxes) - set(a_idxes)) + + a = [x[a_i] for a_i in a_idxes] + b = [x[b_i] for b_i in b_idxes] + + return a, b diff --git a/data_scrapers/daily_quiz/lambda_func.py b/data_scrapers/daily_quiz/lambda_func.py new file mode 100644 index 0000000..4b94605 --- /dev/null +++ b/data_scrapers/daily_quiz/lambda_func.py @@ -0,0 +1,22 @@ +from typing import Any + +from . import protocols +from .aws_lambda.lambda_api import LambdaAPI +from .daily_quiz.serivce import new_daily_quizzes as new_daily_quizzes_ +from .logger import logger +from .steam.steam_api import SteamAPI + + +def new_daily_quizzes(lambda_api: protocols.LambdaAPI, steam_api: protocols.SteamAPI): + logger.info("-- new daily quizzes job start --") + + new_daily_quizzes_(lambda_api, steam_api) + + logger.info("-- new daily quizzes job end --") + + +def lambda_handler(event: Any, context: Any): + lambda_api = LambdaAPI() + steam_api = SteamAPI() + + new_daily_quizzes(lambda_api, steam_api) diff --git a/data_scrapers/daily_quiz/logger.py b/data_scrapers/daily_quiz/logger.py new file mode 100644 index 0000000..9b38413 --- /dev/null +++ b/data_scrapers/daily_quiz/logger.py @@ -0,0 +1,4 @@ +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) diff --git a/data_scrapers/daily_quiz/protocols.py b/data_scrapers/daily_quiz/protocols.py new file mode 100644 index 0000000..d04bc11 --- /dev/null +++ b/data_scrapers/daily_quiz/protocols.py @@ -0,0 +1,20 @@ +from typing import Protocol, Sequence + +from .aws_lambda.model import Game, SaveGameScreenshot, SaveQuiz +from .steam.model import SteamGameScreenshotResponse + + +class SteamAPI(Protocol): + def get_game_screenshots(self, app_id: int, page: int = 1) -> list[SteamGameScreenshotResponse]: + ... + + +class LambdaAPI(Protocol): + def save_screenshots(self, screenshots: Sequence[SaveGameScreenshot]): + ... + + def save_quizzes(self, quizzes: Sequence[SaveQuiz]): + ... + + def get_all_games(self) -> list[Game]: + ... diff --git a/data_scrapers/daily_quiz/steam/model.py b/data_scrapers/daily_quiz/steam/model.py new file mode 100644 index 0000000..beeff8f --- /dev/null +++ b/data_scrapers/daily_quiz/steam/model.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class SteamGameScreenshotResponse(BaseModel): + file_id: int + full_image_url: str diff --git a/data_scrapers/daily_quiz/steam/steam_api.py b/data_scrapers/daily_quiz/steam/steam_api.py new file mode 100644 index 0000000..7ee23ae --- /dev/null +++ b/data_scrapers/daily_quiz/steam/steam_api.py @@ -0,0 +1,15 @@ +from .. import protocols +from . import steampowered_api +from .model import SteamGameScreenshotResponse + + +class SteamAPI(protocols.SteamAPI): + def get_game_screenshots(self, app_id: int, page: int = 1) -> list[SteamGameScreenshotResponse]: + screenshots = steampowered_api.get_community_screenshots(app_id, page) + + return [ + SteamGameScreenshotResponse( + file_id=int(screenshot["published_file_id"]), full_image_url=screenshot["full_image_url"] + ) + for screenshot in screenshots + ] diff --git a/data_scrapers/daily_quiz/steam/steampowered_api.py b/data_scrapers/daily_quiz/steam/steampowered_api.py new file mode 100644 index 0000000..d4722a4 --- /dev/null +++ b/data_scrapers/daily_quiz/steam/steampowered_api.py @@ -0,0 +1,46 @@ +from typing import Any, TypedDict + +import requests + + +class GameScreenshot(TypedDict): + published_file_id: str + type: int + title: str + preview_image_url: str + full_image_url: str + image_width: int + image_height: int + comment_count: int + votes_for: int + content_descriptorids: list + spoiler_tag: Any + description: str + rating_stars: int + maybe_inappropriate_sex: int + maybe_inappropriate_violence: int + youtube_video_id: Any + creator: dict[str, Any] + reactions: list[dict[str, Any]] + + +def get_community_screenshots(app_id: int, page: int) -> list[GameScreenshot]: + res = requests.get(f"https://steamcommunity.com/library/appcommunityfeed/{app_id}?p={page}&rgSections[]=2") + res.raise_for_status() + """ + return example: + ```json + { + "cached": false, + "hub": [ + { + "published_file_id": "3095185741", + "type": 5, + ... + }, + ... + } + ``` + """ + + return res.json()["hub"] From b0e4e79e4a262d1f336ba354c8f2fb253d8711e2 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Mon, 5 Feb 2024 02:24:51 +0900 Subject: [PATCH 04/31] =?UTF-8?q?refactor:=20=ED=95=84=EC=9A=94=EC=97=86?= =?UTF-8?q?=EB=8A=94=20=EC=9D=B4=EB=B2=A4=ED=8A=B8=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/event.py | 1 - data_scrapers/database_lambda/game/service.py | 15 +++----- data_scrapers/database_lambda/lambda_func.py | 7 +--- .../database_lambda/screenshot/service.py | 7 ---- .../game_updater/aws_lambda/event.py | 8 +--- .../game_updater/aws_lambda/lambda_api.py | 26 +------------ .../game_updater/aws_lambda/model.py | 12 +----- data_scrapers/game_updater/lambda_func.py | 11 +----- data_scrapers/game_updater/protocols.py | 15 +------- data_scrapers/game_updater/scraper/model.py | 6 --- .../game_updater/scraper/screenshot.py | 36 ------------------ data_scrapers/game_updater/scraper/service.py | 8 ---- .../database_lambda/game/test_game_service.py | 21 +++------- .../screenshot/test_screenshot_service.py | 12 +----- .../scraper/test_screenshot_scraper.py | 38 ------------------- .../game_updater/utils/mock_lambda_api.py | 16 +------- 16 files changed, 19 insertions(+), 220 deletions(-) delete mode 100644 data_scrapers/game_updater/scraper/screenshot.py delete mode 100644 data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py diff --git a/data_scrapers/database_lambda/event.py b/data_scrapers/database_lambda/event.py index c29e05e..180afc6 100644 --- a/data_scrapers/database_lambda/event.py +++ b/data_scrapers/database_lambda/event.py @@ -4,7 +4,6 @@ "save_games", "save_screenshots", "get_games_in_steam_ids", - "get_some_games", "get_screenshots_in_steam_file_ids", ] diff --git a/data_scrapers/database_lambda/game/service.py b/data_scrapers/database_lambda/game/service.py index 1312ba0..f9cfe47 100644 --- a/data_scrapers/database_lambda/game/service.py +++ b/data_scrapers/database_lambda/game/service.py @@ -11,10 +11,11 @@ genres: dict[str, Genre] = {} -def get_some_games(session: Session) -> list[dict[str, Any]]: - some_games = repository.get_all_games(session) - - return [g.to_dto().model_dump(mode="json") for g in some_games] +# TODO: get_all_games로 바꾸기 +# def get_some_games(session: Session) -> list[dict[str, Any]]: +# some_games = repository.get_all_games(session) +# +# return [g.to_dto().model_dump(mode="json") for g in some_games] def _get_genres(session: Session, name: str) -> Genre: @@ -49,9 +50,3 @@ def save_games(session: Session, games: Sequence[dict[str, Any]]): models[existed.steam_id] = existed session.add_all(models.values()) - - -def get_games_in_steam_ids(session: Session, steam_ids: Sequence[int]) -> list[dict[str, Any]]: - games = repository.get_games_in_steam_ids(session, steam_ids) - - return [g.to_dto().model_dump(mode="json") for g in games] diff --git a/data_scrapers/database_lambda/lambda_func.py b/data_scrapers/database_lambda/lambda_func.py index 560a5ff..54114fb 100644 --- a/data_scrapers/database_lambda/lambda_func.py +++ b/data_scrapers/database_lambda/lambda_func.py @@ -4,16 +4,13 @@ from database_lambda.database import engine, init_database from database_lambda.event import Event, EventName -from database_lambda.game.service import get_games_in_steam_ids, get_some_games, save_games +from database_lambda.game.service import save_games from database_lambda.logger import logger -from database_lambda.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots +from database_lambda.screenshot.service import save_screenshots funcs: dict[EventName, Callable[..., Any]] = { "save_games": save_games, - "get_some_games": get_some_games, - "get_games_in_steam_ids": get_games_in_steam_ids, "save_screenshots": save_screenshots, - "get_screenshots_in_steam_file_ids": get_screenshots_in_steam_file_ids, } diff --git a/data_scrapers/database_lambda/screenshot/service.py b/data_scrapers/database_lambda/screenshot/service.py index 698dae8..fd61a32 100644 --- a/data_scrapers/database_lambda/screenshot/service.py +++ b/data_scrapers/database_lambda/screenshot/service.py @@ -2,16 +2,9 @@ from sqlalchemy.orm import Session -from . import repository from .model import GameScreenshot def save_screenshots(session: Session, screenshots: Collection[dict[str, Any]]): screenshots_ = [GameScreenshot(**s) for s in screenshots] session.add_all(screenshots_) - - -def get_screenshots_in_steam_file_ids(session: Session, file_ids: Collection[int]) -> list[dict[str, Any]]: - screenshots = repository.get_game_screenshots_in_steam_file_ids(session, file_ids) - - return [s.to_dto().model_dump(mode="json") for s in screenshots] diff --git a/data_scrapers/game_updater/aws_lambda/event.py b/data_scrapers/game_updater/aws_lambda/event.py index c29e05e..0ccce19 100644 --- a/data_scrapers/game_updater/aws_lambda/event.py +++ b/data_scrapers/game_updater/aws_lambda/event.py @@ -1,12 +1,6 @@ from typing import Any, Literal, TypedDict -EventName = Literal[ - "save_games", - "save_screenshots", - "get_games_in_steam_ids", - "get_some_games", - "get_screenshots_in_steam_file_ids", -] +EventName = Literal["save_games"] class Event(TypedDict): diff --git a/data_scrapers/game_updater/aws_lambda/lambda_api.py b/data_scrapers/game_updater/aws_lambda/lambda_api.py index bac0522..eaec5ee 100644 --- a/data_scrapers/game_updater/aws_lambda/lambda_api.py +++ b/data_scrapers/game_updater/aws_lambda/lambda_api.py @@ -5,10 +5,9 @@ from .. import protocols from ..config import setting -from ..scraper.model import NewGameScreenshot from .event import Event from .exception import AWSLambdaException -from .model import Game, GameScreenshot, SaveGame +from .model import SaveGame class LambdaAPI(protocols.LambdaAPI): @@ -24,30 +23,7 @@ def invoke_lambda(self, event: Event) -> Any: payload: Any = response["Payload"].read().decode("utf-8") return json.loads(payload) - def get_some_games(self) -> list[Game]: - event = Event(name="get_some_games", payload=None) - - games: list[dict[str, Any]] = self.invoke_lambda(event) - return [Game(**g) for g in games] - - def get_games_in_steam_ids(self, steam_ids: Sequence[int]) -> list[Game]: - event = Event(name="get_games_in_steam_ids", payload=steam_ids) - - games: list[dict[str, Any]] = self.invoke_lambda(event) - return [Game(**g) for g in games] - - def get_screenshots_in_steam_file_ids(self, steam_file_ids: Sequence[int]) -> list[GameScreenshot]: - event = Event(name="get_screenshots_in_steam_file_ids", payload=steam_file_ids) - - screenshots: list[dict[str, Any]] = self.invoke_lambda(event) - return [GameScreenshot(**s) for s in screenshots] - def save_games(self, games: Sequence[SaveGame]): event = Event(name="save_games", payload=[g.model_dump() for g in games]) self.invoke_lambda(event) - - def save_screenshots(self, screenshots: Sequence[NewGameScreenshot]): - event = Event(name="save_screenshots", payload=[s.model_dump() for s in screenshots]) - - self.invoke_lambda(event) diff --git a/data_scrapers/game_updater/aws_lambda/model.py b/data_scrapers/game_updater/aws_lambda/model.py index 5f308cf..8857f70 100644 --- a/data_scrapers/game_updater/aws_lambda/model.py +++ b/data_scrapers/game_updater/aws_lambda/model.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional, Sequence -from pydantic import BaseModel, Field +from pydantic import BaseModel class Game(BaseModel): @@ -15,16 +15,6 @@ class Game(BaseModel): created_at: datetime -class GameScreenshot(BaseModel): - id: int - steam_file_id: int - url: str - game_id: int - game: Game = Field(repr=False) - updated_at: datetime - created_at: datetime - - class SaveGame(BaseModel): steam_id: int name: str diff --git a/data_scrapers/game_updater/lambda_func.py b/data_scrapers/game_updater/lambda_func.py index f67cf02..e6051d5 100644 --- a/data_scrapers/game_updater/lambda_func.py +++ b/data_scrapers/game_updater/lambda_func.py @@ -2,7 +2,7 @@ from game_updater.aws_lambda.lambda_api import LambdaAPI from game_updater.logger import logger -from game_updater.scraper.service import scrap_game_screenshot_for_all, scrap_games +from game_updater.scraper.service import scrap_games from game_updater.steam.steam_api import SteamAPI @@ -14,16 +14,7 @@ def scrap_games_job(lambda_api: LambdaAPI): logger.info("-- scrap game job end -- ") -def scrap_screenshots_job(lambda_api: LambdaAPI): - logger.info("-- scrap screenshot job start -- ") - - scrap_game_screenshot_for_all(SteamAPI(), lambda_api) - - logger.info("-- scrap screenshot job end -- ") - - def lambda_handler(event: Any, context: Any): lambda_api = LambdaAPI() scrap_games_job(lambda_api) - # scrap_screenshots_job(lambda_api) diff --git a/data_scrapers/game_updater/protocols.py b/data_scrapers/game_updater/protocols.py index 0c4e7e4..9e2e9f8 100644 --- a/data_scrapers/game_updater/protocols.py +++ b/data_scrapers/game_updater/protocols.py @@ -1,7 +1,6 @@ from typing import Optional, Protocol, Sequence -from .aws_lambda.model import Game, GameScreenshot, SaveGame -from .scraper.model import NewGameScreenshot +from .aws_lambda.model import SaveGame from .steam.model import ( GamalyticSteamGameDetailResponse, GamalyticSteamGameResponse, @@ -29,17 +28,5 @@ def get_all_games_from_gamalytic(self, worker_cnt: int) -> list[GamalyticSteamGa class LambdaAPI(Protocol): - def get_some_games(self) -> list[Game]: - ... - - def get_games_in_steam_ids(self, steam_ids: Sequence[int]) -> list[Game]: - ... - - def get_screenshots_in_steam_file_ids(self, steam_file_ids: Sequence[int]) -> list[GameScreenshot]: - ... - def save_games(self, games: Sequence[SaveGame]): ... - - def save_screenshots(self, screenshots: Sequence[NewGameScreenshot]): - ... diff --git a/data_scrapers/game_updater/scraper/model.py b/data_scrapers/game_updater/scraper/model.py index e253356..cc94ba1 100644 --- a/data_scrapers/game_updater/scraper/model.py +++ b/data_scrapers/game_updater/scraper/model.py @@ -11,9 +11,3 @@ class Game(BaseModel): genres: Sequence[str] tags: Sequence[str] revenue: float - - -class NewGameScreenshot(BaseModel): - steam_file_id: int - url: str = Field(max_length=2048) - game_id: int diff --git a/data_scrapers/game_updater/scraper/screenshot.py b/data_scrapers/game_updater/scraper/screenshot.py deleted file mode 100644 index a5356c7..0000000 --- a/data_scrapers/game_updater/scraper/screenshot.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Sequence - -from ..aws_lambda.model import Game -from ..logger import logger -from ..protocols import LambdaAPI, SteamAPI -from .model import NewGameScreenshot - - -def _remove_existed_new_screenshot( - lambda_api: LambdaAPI, screenshots: Sequence[NewGameScreenshot] -) -> list[NewGameScreenshot]: - file_id2screenshot = {s.steam_file_id: s for s in screenshots} - exists = set( - s.steam_file_id for s in lambda_api.get_screenshots_in_steam_file_ids(list(file_id2screenshot.keys())) - ) - - return [file_id2screenshot[file_id] for file_id in file_id2screenshot.keys() - exists] - - -def scrap_game_screenshot(steam_api: SteamAPI, lambda_api: LambdaAPI, game: Game) -> None: - # get some screenshots - logger.info("getting some screenshots of game: %s", game) - screenshots: list[NewGameScreenshot] = [] - - for s in steam_api.get_game_screenshots(game.steam_id): - screenshots.append(NewGameScreenshot(steam_file_id=s.file_id, url=s.full_image_url, game_id=game.id)) - logger.info("some screenshots: %s", screenshots) - - # remove existed screenshot - logger.info("removing existed screenshots") - new_screenshots = _remove_existed_new_screenshot(lambda_api, screenshots) - logger.info("remain screenshots: %s", new_screenshots) - - # save new screenshots - lambda_api.save_screenshots(new_screenshots) - logger.info("saved screenshots: %s", new_screenshots) diff --git a/data_scrapers/game_updater/scraper/service.py b/data_scrapers/game_updater/scraper/service.py index 44cecf7..17d26d5 100644 --- a/data_scrapers/game_updater/scraper/service.py +++ b/data_scrapers/game_updater/scraper/service.py @@ -1,14 +1,6 @@ from ..protocols import LambdaAPI, SteamAPI from . import game as game_scraper -from . import screenshot as screenshot_scraper def scrap_games(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: game_scraper.scrap_games(steam_api, lambda_api) - - -def scrap_game_screenshot_for_all(steam_api: SteamAPI, lambda_api: LambdaAPI) -> None: - games = lambda_api.get_some_games() - - for game in games: - screenshot_scraper.scrap_game_screenshot(steam_api, lambda_api, game) diff --git a/data_scrapers/tests/database_lambda/game/test_game_service.py b/data_scrapers/tests/database_lambda/game/test_game_service.py index e482ea8..163a54a 100644 --- a/data_scrapers/tests/database_lambda/game/test_game_service.py +++ b/data_scrapers/tests/database_lambda/game/test_game_service.py @@ -2,16 +2,14 @@ from sqlalchemy.orm import Session from database_lambda.game.model import Game -from database_lambda.game.service import get_games_in_steam_ids, get_some_games, save_games -from tests.database_lambda.utils.game import create_random_game +from database_lambda.game.service import save_games from tests.database_lambda.utils.utils import random_datetime +# def test_get_some_games은_게임을_가져와야한다(session: Session): +# _ = [create_random_game(session) for _ in range(2)] -def test_get_some_games은_게임을_가져와야한다(session: Session): - _ = [create_random_game(session) for _ in range(2)] - - given = get_some_games(session) - assert len(given) == 2 +# given = get_some_games(session) +# assert len(given) == 2 def test_save_games은_입력한_게임을_저장해야_한다(session: Session): @@ -89,12 +87,3 @@ def test_save_games은_이미_저장한_게임은_업데이트_한다(session: S assert saved.kr_name == after_game["kr_name"] assert saved.released_at.timestamp() == after_game["released_at"] assert saved.genres == after_game["genres"] - - -def test_get_games_in_steam_ids은_입력한_스팀_id에_맞는_게임을_가져와야한다(session: Session): - saved_games = [create_random_game(session) for _ in range(2)] - - steam_ids = [saved_games[0].steam_id, saved_games[1].steam_id] - games = get_games_in_steam_ids(session, steam_ids) - - assert set(g["id"] for g in games) == set((saved_games[0].id, saved_games[1].id)) diff --git a/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py index b52b735..2051619 100644 --- a/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py +++ b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py @@ -2,9 +2,8 @@ from sqlalchemy.orm import Session from database_lambda.screenshot.model import GameScreenshot -from database_lambda.screenshot.service import get_screenshots_in_steam_file_ids, save_screenshots +from database_lambda.screenshot.service import save_screenshots from tests.database_lambda.utils.game import create_random_game -from tests.database_lambda.utils.screenshot import create_random_game_screenshot def test_save_screenshots은_입력한_스크린샷을_저장해야_한다(session: Session): @@ -20,12 +19,3 @@ def test_save_screenshots은_입력한_스크린샷을_저장해야_한다(sessi saved = session.scalars(select(GameScreenshot)).all() assert len(saved) == 4 - - -def test_get_screenshots_in_steam_file_ids는_스크린샷을_반환해야_한다(session: Session): - saved_screenshots = [create_random_game_screenshot(session) for _ in range(2)] - - file_ids = [s.steam_file_id for s in saved_screenshots] - - given = get_screenshots_in_steam_file_ids(session, file_ids) - assert len(given) == 2 diff --git a/data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py b/data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py deleted file mode 100644 index 1322d0d..0000000 --- a/data_scrapers/tests/game_updater/scraper/test_screenshot_scraper.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest - -from game_updater.scraper.screenshot import scrap_game_screenshot -from tests.game_updater.utils.mock_lambda_api import MockLambdaAPI -from tests.game_updater.utils.mock_steam_api import MockSteamAPI -from tests.game_updater.utils.model import create_random_game - - -@pytest.fixture -def mock_steam_api() -> MockSteamAPI: - return MockSteamAPI() - - -def test_scrap_screenshots은_스크린샷을_저장한다(mock_steam_api: MockSteamAPI): - mock_steam_api.prepare_mock_data(game_size=1, screenshot_size=1) - saved_game = list(mock_steam_api.games.values())[0] - game = create_random_game(steam_id=saved_game["steam_id"], name=saved_game["name"]) - lambda_api = MockLambdaAPI() - - scrap_game_screenshot(mock_steam_api, lambda_api, game) - - assert len(lambda_api.screenshots.values()) > 0 - - -def test_scrap_screenshot은_중복된_스크린샷은_다시_저장하지_않는다(mock_steam_api: MockSteamAPI): - mock_steam_api.prepare_mock_data(game_size=1, screenshot_size=1) - saved_game = list(mock_steam_api.games.values())[0] - game = create_random_game(steam_id=saved_game["steam_id"], name=saved_game["name"]) - lambda_api = MockLambdaAPI() - - scrap_game_screenshot(mock_steam_api, lambda_api, game) - before_scraped = lambda_api.screenshots.copy() - - # duplicated screenshots - scrap_game_screenshot(mock_steam_api, lambda_api, game) - after_scraped = lambda_api.screenshots - - assert before_scraped == after_scraped diff --git a/data_scrapers/tests/game_updater/utils/mock_lambda_api.py b/data_scrapers/tests/game_updater/utils/mock_lambda_api.py index 81ef076..10678c0 100644 --- a/data_scrapers/tests/game_updater/utils/mock_lambda_api.py +++ b/data_scrapers/tests/game_updater/utils/mock_lambda_api.py @@ -1,23 +1,13 @@ from datetime import datetime from typing import Sequence -from game_updater.aws_lambda.model import Game, GameScreenshot, SaveGame +from game_updater.aws_lambda.model import Game, SaveGame from game_updater.protocols import LambdaAPI class MockLambdaAPI(LambdaAPI): def __init__(self): self.games: dict[int, Game] = {} - self.screenshots: dict[int, GameScreenshot] = {} - - def get_some_games(self) -> list[Game]: - return list(self.games.values()) - - def get_games_in_steam_ids(self, steam_ids: Sequence[int]) -> list[Game]: - return [self.games[id] for id in steam_ids if id in self.games] - - def get_screenshots_in_steam_file_ids(self, steam_file_ids: Sequence[int]) -> list[GameScreenshot]: - return [self.screenshots[id] for id in steam_file_ids if id in self.screenshots] def save_games(self, games: Sequence[SaveGame]): for game in games: @@ -31,7 +21,3 @@ def save_games(self, games: Sequence[SaveGame]): updated_at=datetime.utcnow(), created_at=datetime.utcnow(), ) - - def save_screenshots(self, screenshots: Sequence[GameScreenshot]): - for screenshot in screenshots: - self.screenshots[screenshot.steam_file_id] = screenshot From e7a4f1cbbbe6451a5933a21639815d0066dcc2b7 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Mon, 5 Feb 2024 02:31:21 +0900 Subject: [PATCH 05/31] =?UTF-8?q?feat:=20get=5Fall=5Fgame=20=EC=9D=B4?= =?UTF-8?q?=EB=B2=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/event.py | 7 +------ data_scrapers/database_lambda/game/service.py | 9 ++++----- data_scrapers/database_lambda/lambda_func.py | 7 ++++--- .../tests/database_lambda/game/test_game_service.py | 12 +++++++----- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/data_scrapers/database_lambda/event.py b/data_scrapers/database_lambda/event.py index 180afc6..45346ab 100644 --- a/data_scrapers/database_lambda/event.py +++ b/data_scrapers/database_lambda/event.py @@ -1,11 +1,6 @@ from typing import Any, Literal, TypedDict -EventName = Literal[ - "save_games", - "save_screenshots", - "get_games_in_steam_ids", - "get_screenshots_in_steam_file_ids", -] +EventName = Literal["get_all_games", "save_games", "save_screenshots"] class Event(TypedDict): diff --git a/data_scrapers/database_lambda/game/service.py b/data_scrapers/database_lambda/game/service.py index f9cfe47..6fec17c 100644 --- a/data_scrapers/database_lambda/game/service.py +++ b/data_scrapers/database_lambda/game/service.py @@ -11,11 +11,10 @@ genres: dict[str, Genre] = {} -# TODO: get_all_games로 바꾸기 -# def get_some_games(session: Session) -> list[dict[str, Any]]: -# some_games = repository.get_all_games(session) -# -# return [g.to_dto().model_dump(mode="json") for g in some_games] +def get_all_games(session: Session) -> list[dict[str, Any]]: + some_games = repository.get_all_games(session) + + return [g.to_dto().model_dump(mode="json") for g in some_games] def _get_genres(session: Session, name: str) -> Genre: diff --git a/data_scrapers/database_lambda/lambda_func.py b/data_scrapers/database_lambda/lambda_func.py index 54114fb..10548bc 100644 --- a/data_scrapers/database_lambda/lambda_func.py +++ b/data_scrapers/database_lambda/lambda_func.py @@ -4,22 +4,23 @@ from database_lambda.database import engine, init_database from database_lambda.event import Event, EventName -from database_lambda.game.service import save_games +from database_lambda.game.service import get_all_games, save_games from database_lambda.logger import logger from database_lambda.screenshot.service import save_screenshots funcs: dict[EventName, Callable[..., Any]] = { "save_games": save_games, "save_screenshots": save_screenshots, + "get_all_games": get_all_games, } def handle_event(session: Session, event: Event) -> Any: func = funcs[event["name"]] - try: + if "payload" in event: result = func(session, event["payload"]) - except TypeError: + else: result = func(session) session.commit() diff --git a/data_scrapers/tests/database_lambda/game/test_game_service.py b/data_scrapers/tests/database_lambda/game/test_game_service.py index 163a54a..5ff6237 100644 --- a/data_scrapers/tests/database_lambda/game/test_game_service.py +++ b/data_scrapers/tests/database_lambda/game/test_game_service.py @@ -2,14 +2,16 @@ from sqlalchemy.orm import Session from database_lambda.game.model import Game -from database_lambda.game.service import save_games +from database_lambda.game.service import get_all_games, save_games +from tests.database_lambda.utils.game import create_random_game from tests.database_lambda.utils.utils import random_datetime -# def test_get_some_games은_게임을_가져와야한다(session: Session): -# _ = [create_random_game(session) for _ in range(2)] -# given = get_some_games(session) -# assert len(given) == 2 +def test_get_all_games은_게임을_가져와야한다(session: Session): + saved = [create_random_game(session) for _ in range(2)] + + given = get_all_games(session) + assert set(g["id"] for g in given) == set(g.id for g in saved) def test_save_games은_입력한_게임을_저장해야_한다(session: Session): From 7ae5fe68798ed7ad9ea56d1348d9ed5bc45e06c3 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Mon, 5 Feb 2024 22:33:14 +0900 Subject: [PATCH 06/31] =?UTF-8?q?feat:=20save=5Fscreenshots=20=EA=B8=B0?= =?UTF-8?q?=EB=8A=A5=EC=97=90=EC=84=9C=20=EC=97=85=EB=8D=B0=EC=9D=B4?= =?UTF-8?q?=ED=8A=B8=20=EA=B8=B0=EB=8A=A5=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/event.py | 2 +- .../database_lambda/screenshot/repository.py | 4 +- .../database_lambda/screenshot/service.py | 41 +++++++++++++++++-- .../screenshot/test_screenshot_service.py | 18 +++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/data_scrapers/database_lambda/event.py b/data_scrapers/database_lambda/event.py index 45346ab..a1b8ea8 100644 --- a/data_scrapers/database_lambda/event.py +++ b/data_scrapers/database_lambda/event.py @@ -1,6 +1,6 @@ from typing import Any, Literal, TypedDict -EventName = Literal["get_all_games", "save_games", "save_screenshots"] +EventName = Literal["get_all_games", "save_games", "save_screenshots", "save_quizzes"] class Event(TypedDict): diff --git a/data_scrapers/database_lambda/screenshot/repository.py b/data_scrapers/database_lambda/screenshot/repository.py index 1d426a7..0eebb14 100644 --- a/data_scrapers/database_lambda/screenshot/repository.py +++ b/data_scrapers/database_lambda/screenshot/repository.py @@ -1,4 +1,4 @@ -from typing import Collection, Sequence +from typing import Iterable, Sequence from sqlalchemy import select from sqlalchemy.orm import Session @@ -7,6 +7,6 @@ def get_game_screenshots_in_steam_file_ids( - session: Session, steam_file_ids: Collection[int] + session: Session, steam_file_ids: Iterable[int] ) -> Sequence[GameScreenshot]: return session.scalars(select(GameScreenshot).where(GameScreenshot.steam_file_id.in_(steam_file_ids))).all() diff --git a/data_scrapers/database_lambda/screenshot/service.py b/data_scrapers/database_lambda/screenshot/service.py index fd61a32..dcfc0b7 100644 --- a/data_scrapers/database_lambda/screenshot/service.py +++ b/data_scrapers/database_lambda/screenshot/service.py @@ -1,10 +1,43 @@ -from typing import Any, Collection +from collections.abc import Iterable +from typing import TypedDict from sqlalchemy.orm import Session +from . import repository from .model import GameScreenshot +STEAM_FILE_ID = int -def save_screenshots(session: Session, screenshots: Collection[dict[str, Any]]): - screenshots_ = [GameScreenshot(**s) for s in screenshots] - session.add_all(screenshots_) + +class SaveGameScreenshot(TypedDict): + steam_file_id: STEAM_FILE_ID + url: str + game_id: int + + +def _attach_models(session: Session, models: dict[STEAM_FILE_ID, GameScreenshot]): + saved = repository.get_game_screenshots_in_steam_file_ids(session, models.keys()) + + for s in saved: + query = models[s.steam_file_id] + s.url = query.url + s.game_id = query.game_id + models[s.steam_file_id] = s + + +def _to_models(screenshots: Iterable[SaveGameScreenshot]): + models: dict[STEAM_FILE_ID, GameScreenshot] = {} + for screenshot in screenshots: + models[screenshot["steam_file_id"]] = GameScreenshot(**screenshot) + + return models + + +def save_screenshots(session: Session, screenshots: Iterable[SaveGameScreenshot]): + # turn to model + models = _to_models(screenshots) + + # detached된 객체의 식별자 업데이트 + _attach_models(session, models) + + session.add_all(models.values()) diff --git a/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py index 2051619..03b545f 100644 --- a/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py +++ b/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py @@ -2,14 +2,14 @@ from sqlalchemy.orm import Session from database_lambda.screenshot.model import GameScreenshot -from database_lambda.screenshot.service import save_screenshots +from database_lambda.screenshot.service import SaveGameScreenshot, save_screenshots from tests.database_lambda.utils.game import create_random_game def test_save_screenshots은_입력한_스크린샷을_저장해야_한다(session: Session): saved_games = [create_random_game(session) for _ in range(2)] - screenshots = [ + screenshots: list[SaveGameScreenshot] = [ {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, {"steam_file_id": 2, "url": "https://fake.url/2", "game_id": saved_games[0].id}, {"steam_file_id": 3, "url": "https://fake.url/3", "game_id": saved_games[1].id}, @@ -19,3 +19,17 @@ def test_save_screenshots은_입력한_스크린샷을_저장해야_한다(sessi saved = session.scalars(select(GameScreenshot)).all() assert len(saved) == 4 + + +def test_save_screenshots은_이미_저장한_스크린샷은_업데이트_한다(session: Session): + saved_games = [create_random_game(session) for _ in range(2)] + + before: SaveGameScreenshot = {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id} + after: SaveGameScreenshot = {"steam_file_id": 1, "url": "https://fake.url/1a", "game_id": saved_games[0].id} + + save_screenshots(session, [before]) + save_screenshots(session, [after]) + saved = session.scalars(select(GameScreenshot)).one() + + assert saved.steam_file_id == after["steam_file_id"] + assert saved.url == after["url"] From be89b50ccc2343c6afb6bb25f92ea2f46a49f62d Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Tue, 6 Feb 2024 00:23:14 +0900 Subject: [PATCH 07/31] =?UTF-8?q?refactor:=20database=20lambda=EC=97=90?= =?UTF-8?q?=EC=84=9C=20screeshot=20=EB=A0=88=EC=9D=B4=EC=96=B4=20=EC=97=AD?= =?UTF-8?q?=ED=95=A0=20=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../screenshot/model_factory.py | 33 +++++++++++++++ .../database_lambda/screenshot/schema.py | 9 +++++ .../database_lambda/screenshot/service.py | 40 ++----------------- 3 files changed, 46 insertions(+), 36 deletions(-) create mode 100644 data_scrapers/database_lambda/screenshot/model_factory.py create mode 100644 data_scrapers/database_lambda/screenshot/schema.py diff --git a/data_scrapers/database_lambda/screenshot/model_factory.py b/data_scrapers/database_lambda/screenshot/model_factory.py new file mode 100644 index 0000000..0b711df --- /dev/null +++ b/data_scrapers/database_lambda/screenshot/model_factory.py @@ -0,0 +1,33 @@ +from collections.abc import Iterable + +from sqlalchemy.orm import Session + +from . import repository +from .model import GameScreenshot +from .schema import STEAM_FILE_ID, SaveGameScreenshot + + +def _attach_models(session: Session, models: dict[STEAM_FILE_ID, GameScreenshot]): + saved = repository.get_game_screenshots_in_steam_file_ids(session, models.keys()) + + for s in saved: + query = models[s.steam_file_id] + s.url = query.url + s.game_id = query.game_id + models[s.steam_file_id] = s + + +def _create_models(screenshots: Iterable[SaveGameScreenshot]) -> dict[STEAM_FILE_ID, GameScreenshot]: + models: dict[STEAM_FILE_ID, GameScreenshot] = {} + for screenshot in screenshots: + models[screenshot["steam_file_id"]] = GameScreenshot(**screenshot) + + return models + + +def to_models(session: Session, screenshots: Iterable[SaveGameScreenshot]) -> Iterable[GameScreenshot]: + models = _create_models(screenshots) + # detached된 객체의 식별자 업데이트 + _attach_models(session, models) + + return models.values() diff --git a/data_scrapers/database_lambda/screenshot/schema.py b/data_scrapers/database_lambda/screenshot/schema.py new file mode 100644 index 0000000..f15608e --- /dev/null +++ b/data_scrapers/database_lambda/screenshot/schema.py @@ -0,0 +1,9 @@ +from typing import TypedDict + +STEAM_FILE_ID = int + + +class SaveGameScreenshot(TypedDict): + steam_file_id: STEAM_FILE_ID + url: str + game_id: int diff --git a/data_scrapers/database_lambda/screenshot/service.py b/data_scrapers/database_lambda/screenshot/service.py index dcfc0b7..d690abf 100644 --- a/data_scrapers/database_lambda/screenshot/service.py +++ b/data_scrapers/database_lambda/screenshot/service.py @@ -1,43 +1,11 @@ from collections.abc import Iterable -from typing import TypedDict from sqlalchemy.orm import Session -from . import repository -from .model import GameScreenshot - -STEAM_FILE_ID = int - - -class SaveGameScreenshot(TypedDict): - steam_file_id: STEAM_FILE_ID - url: str - game_id: int - - -def _attach_models(session: Session, models: dict[STEAM_FILE_ID, GameScreenshot]): - saved = repository.get_game_screenshots_in_steam_file_ids(session, models.keys()) - - for s in saved: - query = models[s.steam_file_id] - s.url = query.url - s.game_id = query.game_id - models[s.steam_file_id] = s - - -def _to_models(screenshots: Iterable[SaveGameScreenshot]): - models: dict[STEAM_FILE_ID, GameScreenshot] = {} - for screenshot in screenshots: - models[screenshot["steam_file_id"]] = GameScreenshot(**screenshot) - - return models +from .model_factory import to_models +from .schema import SaveGameScreenshot def save_screenshots(session: Session, screenshots: Iterable[SaveGameScreenshot]): - # turn to model - models = _to_models(screenshots) - - # detached된 객체의 식별자 업데이트 - _attach_models(session, models) - - session.add_all(models.values()) + models = to_models(session, screenshots) + session.add_all(models) From 3cd505b6ad6d9ae0f84893aa2e77b0ac6a3f764b Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Tue, 6 Feb 2024 01:07:34 +0900 Subject: [PATCH 08/31] =?UTF-8?q?feat:=20save=5Fquizzes=20=EC=9D=B4?= =?UTF-8?q?=EB=B2=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/quiz/model.py | 31 +++++++++++++++++++ .../database_lambda/quiz/model_factory.py | 16 ++++++++++ data_scrapers/database_lambda/quiz/schema.py | 8 +++++ data_scrapers/database_lambda/quiz/service.py | 11 +++++++ .../screenshot/model_factory.py | 4 +-- 5 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 data_scrapers/database_lambda/quiz/model.py create mode 100644 data_scrapers/database_lambda/quiz/model_factory.py create mode 100644 data_scrapers/database_lambda/quiz/schema.py create mode 100644 data_scrapers/database_lambda/quiz/service.py diff --git a/data_scrapers/database_lambda/quiz/model.py b/data_scrapers/database_lambda/quiz/model.py new file mode 100644 index 0000000..ff79c73 --- /dev/null +++ b/data_scrapers/database_lambda/quiz/model.py @@ -0,0 +1,31 @@ +from collections.abc import Sequence + +from pydantic import BaseModel +from sqlalchemy import Column, ForeignKey, Integer, Table +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from ..model import Base, CreatedAtMixin, UpdatedAtMixin +from ..screenshot.model import GameScreenshot, GameScreenshotDto + +quiz_screenshot_link = Table( + "quiz_screenshot_link", + Base.metadata, + Column("id", type_=Integer, primary_key=True), + Column("quiz_id", ForeignKey("quiz.id")), + Column("screenshot_id", ForeignKey("game_screenshot.id")), +) + + +class Quiz(CreatedAtMixin, UpdatedAtMixin, Base): + __tablename__ = "quiz" + + id: Mapped[int] = mapped_column(primary_key=True) + screenshots: Mapped[list[GameScreenshot]] = relationship(secondary=quiz_screenshot_link) + + def to_dto(self) -> "QuizDto": + return QuizDto(id=self.id, screenshots=[s.to_dto() for s in self.screenshots]) + + +class QuizDto(BaseModel): + id: int + screenshots: Sequence[GameScreenshotDto] diff --git a/data_scrapers/database_lambda/quiz/model_factory.py b/data_scrapers/database_lambda/quiz/model_factory.py new file mode 100644 index 0000000..5299be5 --- /dev/null +++ b/data_scrapers/database_lambda/quiz/model_factory.py @@ -0,0 +1,16 @@ +from collections.abc import Iterable + +from sqlalchemy.orm import Session + +from ..screenshot import model_factory as screenshot_model_factory +from .model import Quiz +from .schema import SaveQuiz + + +def to_models(session: Session, quizzes: Iterable[SaveQuiz]) -> list[Quiz]: + models: list[Quiz] = [] + for quiz in quizzes: + screenshot_models = screenshot_model_factory.to_models(session, screenshots=quiz["screenshots"]) + models.append(Quiz(screenshots=screenshot_models)) + + return models diff --git a/data_scrapers/database_lambda/quiz/schema.py b/data_scrapers/database_lambda/quiz/schema.py new file mode 100644 index 0000000..971d544 --- /dev/null +++ b/data_scrapers/database_lambda/quiz/schema.py @@ -0,0 +1,8 @@ +from collections.abc import Sequence +from typing import TypedDict + +from ..screenshot.schema import SaveGameScreenshot + + +class SaveQuiz(TypedDict): + screenshots: Sequence[SaveGameScreenshot] diff --git a/data_scrapers/database_lambda/quiz/service.py b/data_scrapers/database_lambda/quiz/service.py new file mode 100644 index 0000000..6acd6ab --- /dev/null +++ b/data_scrapers/database_lambda/quiz/service.py @@ -0,0 +1,11 @@ +from collections.abc import Iterable + +from sqlalchemy.orm import Session + +from .model_factory import to_models +from .schema import SaveQuiz + + +def save_quizzes(session: Session, quizzes: Iterable[SaveQuiz]): + models = to_models(session, quizzes) + session.add_all(models) diff --git a/data_scrapers/database_lambda/screenshot/model_factory.py b/data_scrapers/database_lambda/screenshot/model_factory.py index 0b711df..f6bb9c5 100644 --- a/data_scrapers/database_lambda/screenshot/model_factory.py +++ b/data_scrapers/database_lambda/screenshot/model_factory.py @@ -25,9 +25,9 @@ def _create_models(screenshots: Iterable[SaveGameScreenshot]) -> dict[STEAM_FILE return models -def to_models(session: Session, screenshots: Iterable[SaveGameScreenshot]) -> Iterable[GameScreenshot]: +def to_models(session: Session, screenshots: Iterable[SaveGameScreenshot]) -> list[GameScreenshot]: models = _create_models(screenshots) # detached된 객체의 식별자 업데이트 _attach_models(session, models) - return models.values() + return list(models.values()) From ad57c3ceb07983e43159b309f28a9e6b521fcb07 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Tue, 6 Feb 2024 01:10:20 +0900 Subject: [PATCH 09/31] =?UTF-8?q?test:=20save=5Fquizzes=20=EC=9D=B4?= =?UTF-8?q?=EB=B2=A4=ED=8A=B8=EC=97=90=20=EB=8C=80=ED=95=9C=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=BD=94=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database_lambda/quiz/test_quiz_service.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 data_scrapers/tests/database_lambda/quiz/test_quiz_service.py diff --git a/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py b/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py new file mode 100644 index 0000000..506f0ec --- /dev/null +++ b/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py @@ -0,0 +1,72 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session + +from database_lambda.quiz.model import Quiz +from database_lambda.quiz.schema import SaveQuiz +from database_lambda.quiz.service import save_quizzes +from tests.database_lambda.utils.game import create_random_game + + +def test_save_quizzes은_입력한_퀴즈를_저장해야_한다(session: Session): + saved_games = [create_random_game(session) for _ in range(2)] + quizzes: list[SaveQuiz] = [ + { + "screenshots": [ + {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, + {"steam_file_id": 2, "url": "https://fake.url/2", "game_id": saved_games[0].id}, + {"steam_file_id": 3, "url": "https://fake.url/3", "game_id": saved_games[0].id}, + {"steam_file_id": 4, "url": "https://fake.url/4", "game_id": saved_games[0].id}, + {"steam_file_id": 5, "url": "https://fake.url/5", "game_id": saved_games[0].id}, + ] + }, + { + "screenshots": [ + {"steam_file_id": 6, "url": "https://fake.url/6", "game_id": saved_games[1].id}, + {"steam_file_id": 7, "url": "https://fake.url/7", "game_id": saved_games[1].id}, + {"steam_file_id": 8, "url": "https://fake.url/8", "game_id": saved_games[1].id}, + {"steam_file_id": 9, "url": "https://fake.url/9", "game_id": saved_games[1].id}, + {"steam_file_id": 10, "url": "https://fake.url/10", "game_id": saved_games[1].id}, + ] + }, + ] + + save_quizzes(session, quizzes) + + saved = session.scalars(select(Quiz)).all() + assert len(saved) == 2 + + +def test_save_quizzes은_입력한_퀴즈내_스크린샷을_저장해야_한다(session: Session): + saved_games = [create_random_game(session) for _ in range(2)] + quizzes: list[SaveQuiz] = [ + { + "screenshots": [ + {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, + {"steam_file_id": 2, "url": "https://fake.url/2", "game_id": saved_games[0].id}, + {"steam_file_id": 3, "url": "https://fake.url/3", "game_id": saved_games[0].id}, + {"steam_file_id": 4, "url": "https://fake.url/4", "game_id": saved_games[0].id}, + {"steam_file_id": 5, "url": "https://fake.url/5", "game_id": saved_games[0].id}, + ] + }, + { + "screenshots": [ + {"steam_file_id": 6, "url": "https://fake.url/6", "game_id": saved_games[1].id}, + {"steam_file_id": 7, "url": "https://fake.url/7", "game_id": saved_games[1].id}, + {"steam_file_id": 8, "url": "https://fake.url/8", "game_id": saved_games[1].id}, + {"steam_file_id": 9, "url": "https://fake.url/9", "game_id": saved_games[1].id}, + {"steam_file_id": 10, "url": "https://fake.url/10", "game_id": saved_games[1].id}, + ] + }, + ] + + save_quizzes(session, quizzes) + + saved = session.scalars(select(Quiz)).all() + for saved_q, q in zip(saved, quizzes): + saved_screenshots = sorted(saved_q.screenshots, key=lambda s: s.steam_file_id) + screenshots = sorted(q["screenshots"], key=lambda s: s["steam_file_id"]) + + for saved_s, s in zip(saved_screenshots, screenshots): + assert saved_s.steam_file_id == s["steam_file_id"] + assert saved_s.game_id == s["game_id"] + assert saved_s.url == s["url"] From b8051dbd51adc06faa9ab5f5b67174fb7de3e526 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Tue, 6 Feb 2024 01:27:06 +0900 Subject: [PATCH 10/31] =?UTF-8?q?test:=20=ED=95=9C=20=ED=80=B4=EC=A6=88?= =?UTF-8?q?=EC=97=90=20=EC=A4=91=EB=B3=B5=EB=90=9C=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A6=B0=EC=83=B7,=20=EC=97=AC=EB=9F=AC=EA=B0=9C=EC=9D=98=20?= =?UTF-8?q?=EA=B2=8C=EC=9E=84=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=BC=80?= =?UTF-8?q?=EC=9D=B4=EC=8A=A4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database_lambda/quiz/exception.py | 6 +++ .../database_lambda/quiz/test_quiz_service.py | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 data_scrapers/database_lambda/quiz/exception.py diff --git a/data_scrapers/database_lambda/quiz/exception.py b/data_scrapers/database_lambda/quiz/exception.py new file mode 100644 index 0000000..d89a509 --- /dev/null +++ b/data_scrapers/database_lambda/quiz/exception.py @@ -0,0 +1,6 @@ +class MultipleGamesInQuizError(Exception): + pass + + +class DuplicatedScreenshotsInQuizError(Exception): + pass diff --git a/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py b/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py index 506f0ec..8202ca4 100644 --- a/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py +++ b/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py @@ -1,6 +1,8 @@ +import pytest from sqlalchemy import select from sqlalchemy.orm import Session +from database_lambda.quiz.exception import DuplicatedScreenshotsInQuizError, MultipleGamesInQuizError from database_lambda.quiz.model import Quiz from database_lambda.quiz.schema import SaveQuiz from database_lambda.quiz.service import save_quizzes @@ -70,3 +72,39 @@ def test_save_quizzes은_입력한_퀴즈내_스크린샷을_저장해야_한다 assert saved_s.steam_file_id == s["steam_file_id"] assert saved_s.game_id == s["game_id"] assert saved_s.url == s["url"] + + +def test_save_quizzes은_한_퀴즈에_여러개의_게임_스크린샷이_있으면_예외를_던져야_한다(session: Session): + saved_games = [create_random_game(session) for _ in range(2)] + quizzes: list[SaveQuiz] = [ + { + "screenshots": [ + {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, + {"steam_file_id": 2, "url": "https://fake.url/2", "game_id": saved_games[0].id}, + {"steam_file_id": 3, "url": "https://fake.url/3", "game_id": saved_games[1].id}, + {"steam_file_id": 4, "url": "https://fake.url/4", "game_id": saved_games[1].id}, + {"steam_file_id": 5, "url": "https://fake.url/5", "game_id": saved_games[1].id}, + ] + } + ] + + with pytest.raises(MultipleGamesInQuizError): + save_quizzes(session, quizzes) + + +def test_save_quizzes은_한_퀴즈에_중복된_스크린샷이_있으면_예외를_던져야_한다(session: Session): + saved_games = [create_random_game(session) for _ in range(2)] + quizzes: list[SaveQuiz] = [ + { + "screenshots": [ + {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, + {"steam_file_id": 1, "url": "https://fake.url/1", "game_id": saved_games[0].id}, + {"steam_file_id": 3, "url": "https://fake.url/3", "game_id": saved_games[0].id}, + {"steam_file_id": 4, "url": "https://fake.url/4", "game_id": saved_games[0].id}, + {"steam_file_id": 5, "url": "https://fake.url/5", "game_id": saved_games[0].id}, + ] + } + ] + + with pytest.raises(DuplicatedScreenshotsInQuizError): + save_quizzes(session, quizzes) From 18405fa9616fac6550c27e71986458f28eb03ec9 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 16:56:22 +0900 Subject: [PATCH 11/31] =?UTF-8?q?feat:=20=ED=80=B4=EC=A6=88=EB=82=B4=20?= =?UTF-8?q?=EC=A4=91=EB=B3=B5=EB=90=9C=20=EC=8A=A4=ED=81=AC=EB=A6=B0?= =?UTF-8?q?=EC=83=B7=20=EA=B2=80=EC=82=AC=20=EA=B8=B0=EB=8A=A5=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/quiz/service.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/data_scrapers/database_lambda/quiz/service.py b/data_scrapers/database_lambda/quiz/service.py index 6acd6ab..e419ef9 100644 --- a/data_scrapers/database_lambda/quiz/service.py +++ b/data_scrapers/database_lambda/quiz/service.py @@ -2,10 +2,20 @@ from sqlalchemy.orm import Session +from .exception import DuplicatedScreenshotsInQuizError from .model_factory import to_models from .schema import SaveQuiz def save_quizzes(session: Session, quizzes: Iterable[SaveQuiz]): + validate_duplicated_screenshots(quizzes) + models = to_models(session, quizzes) session.add_all(models) + + +def validate_duplicated_screenshots(quizzes: Iterable[SaveQuiz]): + for quiz in quizzes: + screenshot_identities = [s["steam_file_id"] for s in quiz["screenshots"]] + if len(set(screenshot_identities)) < len(screenshot_identities): + raise DuplicatedScreenshotsInQuizError(f"중복된 스크린샷 식별자가 검출됨, 식별자: {screenshot_identities}") From 58a1fb2c3940aa6c538682b1884efcf8819ad65a Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 16:59:17 +0900 Subject: [PATCH 12/31] =?UTF-8?q?feat:=20=ED=80=B4=EC=A6=88=EB=82=B4=20?= =?UTF-8?q?=EB=8B=A4=EC=88=98=EC=9D=98=20=EA=B2=8C=EC=9E=84=EC=9D=98=20?= =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=B0=EC=83=B7=EC=9D=B4=20=EA=B2=80?= =?UTF-8?q?=EC=B6=9C=EB=90=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/database_lambda/quiz/service.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/data_scrapers/database_lambda/quiz/service.py b/data_scrapers/database_lambda/quiz/service.py index e419ef9..f0ac222 100644 --- a/data_scrapers/database_lambda/quiz/service.py +++ b/data_scrapers/database_lambda/quiz/service.py @@ -2,20 +2,27 @@ from sqlalchemy.orm import Session -from .exception import DuplicatedScreenshotsInQuizError +from .exception import DuplicatedScreenshotsInQuizError, MultipleGamesInQuizError from .model_factory import to_models from .schema import SaveQuiz def save_quizzes(session: Session, quizzes: Iterable[SaveQuiz]): - validate_duplicated_screenshots(quizzes) + for quiz in quizzes: + validate_duplicated_screenshots(quiz) + validate_multiple_games(quiz) models = to_models(session, quizzes) session.add_all(models) -def validate_duplicated_screenshots(quizzes: Iterable[SaveQuiz]): - for quiz in quizzes: - screenshot_identities = [s["steam_file_id"] for s in quiz["screenshots"]] - if len(set(screenshot_identities)) < len(screenshot_identities): - raise DuplicatedScreenshotsInQuizError(f"중복된 스크린샷 식별자가 검출됨, 식별자: {screenshot_identities}") +def validate_duplicated_screenshots(quiz: SaveQuiz): + screenshot_identities = [s["steam_file_id"] for s in quiz["screenshots"]] + if len(set(screenshot_identities)) < len(screenshot_identities): + raise DuplicatedScreenshotsInQuizError(f"중복된 스크린샷 식별자가 검출됨, 식별자: {screenshot_identities}") + + +def validate_multiple_games(quiz: SaveQuiz): + game_ids = [s["game_id"] for s in quiz["screenshots"]] + if len(set(game_ids)) > 1: + raise MultipleGamesInQuizError(f"한 퀴즈내에 여러개의 게임이 검출됨, 게임 id: {game_ids}") From dc850a29b7e268597bcf8d6295f892e7f061ee9c Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 17:37:04 +0900 Subject: [PATCH 13/31] =?UTF-8?q?refactor:=20=EC=84=A4=EC=A0=95=EA=B0=92?= =?UTF-8?q?=20=ED=83=80=EC=9E=85=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_scrapers/daily_quiz/config.py b/data_scrapers/daily_quiz/config.py index 24e4e8b..b833473 100644 --- a/data_scrapers/daily_quiz/config.py +++ b/data_scrapers/daily_quiz/config.py @@ -1,10 +1,12 @@ +from collections.abc import Sequence + from pydantic_settings import BaseSettings, SettingsConfigDict class Config(BaseSettings): DATABASE_LAMBDA_NAME: str = "database" DAILY_QUIZ_CNT: int = 5 - GAME_GENERES: list[str] = [ + GAME_GENERES: Sequence[str] = [ "Action", "Adventure", "Massively Multiplayer", From fdcd360ba597f3d58b77226708ac16f1c27a836a Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 17:39:12 +0900 Subject: [PATCH 14/31] =?UTF-8?q?test:=20genre=5Fpicker=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=9E=91=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/test_game_picker.py | 2 ++ .../daily_quiz/test_genre_picker.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py create mode 100644 data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py new file mode 100644 index 0000000..0205108 --- /dev/null +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py @@ -0,0 +1,2 @@ +def test_pick_games은(): + dfse diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py new file mode 100644 index 0000000..eca487a --- /dev/null +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py @@ -0,0 +1,20 @@ +import pytest + +from daily_quiz.config import setting +from daily_quiz.daily_quiz.genre_picker import pick_genres + + +@pytest.mark.parametrize("setting_quiz_cnt", (5, 10)) +def test_pick_genres는_퀴즈의_개수만큼_장르를_반환해야_한다(setting_quiz_cnt: int): + setting.DAILY_QUIZ_CNT = setting_quiz_cnt + genres = pick_genres() + + assert len(genres) == setting.DAILY_QUIZ_CNT + + +@pytest.mark.parametrize("setting_genres", (tuple(setting.GAME_GENERES), ("Adventure", "RPG"))) +def test_pick_genres는_설정값에서_장르를_선택해야_한다(setting_genres: tuple[str]): + setting.GAME_GENERES = setting_genres + genres = pick_genres() + + assert set(setting.GAME_GENERES) == set(genres) | set(setting.GAME_GENERES) From 12c457ac4d954fc41cc3b0dac5c8138bb2c18659 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 17:45:14 +0900 Subject: [PATCH 15/31] =?UTF-8?q?fix:=20=ED=80=B4=EC=A6=88=20=EA=B0=9C?= =?UTF-8?q?=EC=88=98=EB=A7=8C=ED=81=BC=20=EC=9E=A5=EB=A5=B4=EB=A5=BC=20?= =?UTF-8?q?=EC=84=A0=ED=83=9D=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/genre_picker.py | 4 ++-- data_scrapers/daily_quiz/daily_quiz/serivce.py | 3 ++- .../daily_quiz/daily_quiz/test_genre_picker.py | 14 +++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py index 1c5f0fd..17cfbc3 100644 --- a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py +++ b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py @@ -3,5 +3,5 @@ from ..config import setting -def pick_genres() -> list[str]: - return random.choices(setting.GAME_GENERES, k=5) +def pick_genres(k: int) -> list[str]: + return random.choices(setting.GAME_GENERES, k=k) diff --git a/data_scrapers/daily_quiz/daily_quiz/serivce.py b/data_scrapers/daily_quiz/daily_quiz/serivce.py index 4bab0c8..9e5eda7 100644 --- a/data_scrapers/daily_quiz/daily_quiz/serivce.py +++ b/data_scrapers/daily_quiz/daily_quiz/serivce.py @@ -1,6 +1,7 @@ from typing import Iterable from ..aws_lambda.model import Game, SaveQuiz +from ..config import setting from ..protocols import LambdaAPI, SteamAPI from .game_picker import pick_games from .genre_picker import pick_genres @@ -22,7 +23,7 @@ def new_daily_quizzes(lambda_api: LambdaAPI, steam_api: SteamAPI): all_games = lambda_api.get_all_games() # 장르 선택 - picked_genres = pick_genres() + picked_genres = pick_genres(setting.DAILY_QUIZ_CNT) # 게임 선택 picked_games = pick_games(all_games, picked_genres) diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py index eca487a..2616423 100644 --- a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py @@ -4,17 +4,17 @@ from daily_quiz.daily_quiz.genre_picker import pick_genres -@pytest.mark.parametrize("setting_quiz_cnt", (5, 10)) -def test_pick_genres는_퀴즈의_개수만큼_장르를_반환해야_한다(setting_quiz_cnt: int): - setting.DAILY_QUIZ_CNT = setting_quiz_cnt - genres = pick_genres() +@pytest.mark.parametrize("genre_cnt", (5, 10)) +def test_pick_genres는_입력된_개수만큼_장르를_반환해야_한다(genre_cnt: int): + genres = pick_genres(genre_cnt) - assert len(genres) == setting.DAILY_QUIZ_CNT + assert len(genres) == genre_cnt @pytest.mark.parametrize("setting_genres", (tuple(setting.GAME_GENERES), ("Adventure", "RPG"))) -def test_pick_genres는_설정값에서_장르를_선택해야_한다(setting_genres: tuple[str]): +@pytest.mark.parametrize("genre_cnt", (5, 10)) +def test_pick_genres는_설정값에서_장르를_선택해야_한다(genre_cnt: int, setting_genres: tuple[str]): setting.GAME_GENERES = setting_genres - genres = pick_genres() + genres = pick_genres(genre_cnt) assert set(setting.GAME_GENERES) == set(genres) | set(setting.GAME_GENERES) From 1fa5f94b23dd70dc06231a6ee4954baa6134d574 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 19:14:21 +0900 Subject: [PATCH 16/31] =?UTF-8?q?test:=20game=5Fpicker=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/test_game_picker.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py index 0205108..30234f6 100644 --- a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py @@ -1,2 +1,24 @@ -def test_pick_games은(): - dfse +from daily_quiz.daily_quiz.game_picker import pick_games +from tests.daily_quiz.utils.model import create_random_game + + +def test_pick_games은_입력한_게임에서_장르별로_1개씩_선택해야_한다(): + genres = [ + "Action", + "Adventure", + "Massively Multiplayer", + "Strategy", + "RPG", + ] + games = [ + create_random_game(genres=["Action", "Adventure"]), + create_random_game(genres=["Action", "Massively Multiplayer"]), + create_random_game(genres=["Adventure", "Massively Multiplayer"]), + create_random_game(genres=["Adventure", "Strategy"]), + create_random_game(genres=["Massively Multiplayer", "Strategy"]), + create_random_game(genres=["Strategy", "RPG"]), + ] + + picked = pick_games(games, genres) + + assert len(picked) == len(genres) From f2a9e10e02f10e5f014dcd2a72247b827e99c6da Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 19:14:45 +0900 Subject: [PATCH 17/31] =?UTF-8?q?fix:=20Game=EC=97=90=20=ED=95=B4=EC=8B=9C?= =?UTF-8?q?=ED=95=A8=EC=88=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/aws_lambda/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data_scrapers/daily_quiz/aws_lambda/model.py b/data_scrapers/daily_quiz/aws_lambda/model.py index aaed741..17a7de8 100644 --- a/data_scrapers/daily_quiz/aws_lambda/model.py +++ b/data_scrapers/daily_quiz/aws_lambda/model.py @@ -14,6 +14,9 @@ class Game(BaseModel): updated_at: datetime created_at: datetime + def __hash__(self) -> int: + return hash(self.id) + class SaveGameScreenshot(BaseModel): steam_file_id: int From a5a64ad2e4e3f20a1f95c23294fd5e5b4c065276 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 22:53:15 +0900 Subject: [PATCH 18/31] =?UTF-8?q?test:=20new=5Fdaily=5Fquizzes=EC=97=90=20?= =?UTF-8?q?=EB=8C=80=ED=95=9C=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/test_new_daily_quizzes.py | 17 +++++++ .../tests/daily_quiz/utils/mock_lambda_api.py | 24 ++++++++++ .../tests/daily_quiz/utils/mock_steam_api.py | 24 ++++++++++ data_scrapers/tests/daily_quiz/utils/model.py | 45 +++++++++++++++++++ data_scrapers/tests/daily_quiz/utils/steam.py | 8 ++++ data_scrapers/tests/daily_quiz/utils/utils.py | 6 +++ 6 files changed, 124 insertions(+) create mode 100644 data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py create mode 100644 data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py create mode 100644 data_scrapers/tests/daily_quiz/utils/mock_steam_api.py create mode 100644 data_scrapers/tests/daily_quiz/utils/model.py create mode 100644 data_scrapers/tests/daily_quiz/utils/steam.py create mode 100644 data_scrapers/tests/daily_quiz/utils/utils.py diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py new file mode 100644 index 0000000..f54c959 --- /dev/null +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py @@ -0,0 +1,17 @@ +import pytest + +from daily_quiz.config import setting +from daily_quiz.daily_quiz.serivce import new_daily_quizzes +from tests.daily_quiz.utils.mock_lambda_api import MockLambdaAPI +from tests.daily_quiz.utils.mock_steam_api import MockSteamAPI + + +@pytest.mark.parametrize("setting_quiz_cnt", (5, 10)) +def test_new_daily_quizzes는_설정값만큼_퀴즈를_만들어야한다(setting_quiz_cnt: int): + setting.DAILY_QUIZ_CNT = setting_quiz_cnt + lambda_api = MockLambdaAPI() + steam_api = MockSteamAPI() + + new_daily_quizzes(lambda_api, steam_api) + + assert len(lambda_api.quizzes) == setting_quiz_cnt diff --git a/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py b/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py new file mode 100644 index 0000000..93dbefa --- /dev/null +++ b/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py @@ -0,0 +1,24 @@ +from random import choices +from typing import Sequence + +from daily_quiz.aws_lambda.model import Game, SaveGameScreenshot, SaveQuiz +from daily_quiz.config import setting +from daily_quiz.protocols import LambdaAPI +from tests.daily_quiz.utils.model import create_random_game + + +class MockLambdaAPI(LambdaAPI): + def __init__(self) -> None: + self.games = [] + for k in range(1, 4): + self.games.extend([create_random_game(genres=choices(setting.GAME_GENERES, k=k)) for i in range(100)]) + self.quizzes = [] + + def save_quizzes(self, quizzes: Sequence[SaveQuiz]): + self.quizzes.extend(quizzes) + + def get_all_games(self) -> list[Game]: + return self.games + + def save_screenshots(self, screenshots: Sequence[SaveGameScreenshot]): + pass diff --git a/data_scrapers/tests/daily_quiz/utils/mock_steam_api.py b/data_scrapers/tests/daily_quiz/utils/mock_steam_api.py new file mode 100644 index 0000000..af7093e --- /dev/null +++ b/data_scrapers/tests/daily_quiz/utils/mock_steam_api.py @@ -0,0 +1,24 @@ +from daily_quiz.protocols import SteamAPI +from daily_quiz.steam.model import SteamGameScreenshotResponse + +from ..utils.steam import create_random_screenshot + +MOCK_GAME_SIZE = 100 +FEATURE_GAME_SIZE = 100 +MOCK_GENRE_SIZE = 10 +MOCK_SCREENSHOT_SIZE = 1000 +SCREENSHOT_PAGE_SIZE = 100 + + +class MockSteamAPI(SteamAPI): + def __init__(self) -> None: + self.screenshots: dict[int, list[dict]] = {} + + def get_game_screenshots(self, app_id: int, page: int = 1) -> list[SteamGameScreenshotResponse]: + if app_id not in self.screenshots: + self.screenshots[app_id] = [create_random_screenshot() for _ in range(100)] + + return [ + SteamGameScreenshotResponse(file_id=s["file_id"], full_image_url=s["url"]) + for s in self.screenshots[app_id][SCREENSHOT_PAGE_SIZE * (page - 1) : SCREENSHOT_PAGE_SIZE * page] + ] diff --git a/data_scrapers/tests/daily_quiz/utils/model.py b/data_scrapers/tests/daily_quiz/utils/model.py new file mode 100644 index 0000000..3ea1fbe --- /dev/null +++ b/data_scrapers/tests/daily_quiz/utils/model.py @@ -0,0 +1,45 @@ +from datetime import datetime +from typing import Optional + +from daily_quiz.aws_lambda.model import Game + +from .utils import random_datetime + +game_id_counter = 1 + + +def create_random_game( + *, + steam_id: Optional[int] = None, + name: Optional[str] = None, + kr_name: Optional[str] = None, + genres: Optional[list[str]] = None, + released_at: Optional[datetime] = None, +) -> Game: + global game_id_counter + game_id_counter += 1 + + if steam_id is None: + steam_id = game_id_counter + if name is None: + name = f"Game #{steam_id}" + if kr_name is None: + kr_name = f"게임 #{steam_id}" + if genres is None: + genres = [f"Genre #{steam_id}"] + if released_at is None: + released_at = random_datetime() + + created_at = datetime.utcnow() + updated_at = datetime.utcnow() + + return Game( + id=game_id_counter, + steam_id=steam_id, + name=name, + kr_name=kr_name, + created_at=created_at, + updated_at=updated_at, + genres=genres, + released_at=released_at, + ) diff --git a/data_scrapers/tests/daily_quiz/utils/steam.py b/data_scrapers/tests/daily_quiz/utils/steam.py new file mode 100644 index 0000000..91a3bdc --- /dev/null +++ b/data_scrapers/tests/daily_quiz/utils/steam.py @@ -0,0 +1,8 @@ +screenshot_file_id_counter = 1 + + +def create_random_screenshot(): + global screenshot_file_id_counter + screenshot_file_id_counter += 1 + + return {"file_id": screenshot_file_id_counter, "url": f"https://example.com/file/{screenshot_file_id_counter}"} diff --git a/data_scrapers/tests/daily_quiz/utils/utils.py b/data_scrapers/tests/daily_quiz/utils/utils.py new file mode 100644 index 0000000..d7c39d5 --- /dev/null +++ b/data_scrapers/tests/daily_quiz/utils/utils.py @@ -0,0 +1,6 @@ +from datetime import datetime +from random import randint + + +def random_datetime() -> datetime: + return datetime(year=randint(1999, 2024), month=randint(1, 12), day=randint(1, 20)) From 9a71b03e559f726673a1b9c7b080f21b63271237 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 22:54:00 +0900 Subject: [PATCH 19/31] =?UTF-8?q?fix:=20game=20picker=EC=97=90=EC=84=9C=20?= =?UTF-8?q?=EB=8D=94=20=ED=9A=A8=EC=9C=A8=EC=A0=81=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EA=B2=8C=EC=9E=84=EC=9D=84=20=EC=84=A0=EC=A0=95=ED=95=98?= =?UTF-8?q?=EB=8F=84=EB=A1=9D=20=EC=88=98=EC=A0=95,=20=EB=82=A0=EC=A7=9C?= =?UTF-8?q?=20=EA=B8=B0=EB=B0=98=20=EC=84=A0=ED=83=9D=20=EB=AF=B8=EC=A0=81?= =?UTF-8?q?=EC=9A=A9=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/daily_quiz/game_picker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_scrapers/daily_quiz/daily_quiz/game_picker.py b/data_scrapers/daily_quiz/daily_quiz/game_picker.py index 6ba6b3d..9c5e706 100644 --- a/data_scrapers/daily_quiz/daily_quiz/game_picker.py +++ b/data_scrapers/daily_quiz/daily_quiz/game_picker.py @@ -51,7 +51,7 @@ def _pick_older_newer_games( def _pick_unique_per_category(categorized_games: Iterable[GameGroup]) -> set[Game]: unique_games: set[Game] = set() - for games in categorized_games: + for games in sorted(categorized_games, key=len): games_ = list(set(games) - unique_games) game = random.choice(games_) unique_games.add(game) @@ -62,7 +62,7 @@ def _pick_unique_per_category(categorized_games: Iterable[GameGroup]) -> set[Gam def pick_games( games: Iterable[Game], genres: Iterable[str], -): +) -> set[Game]: categorized_games = _categorize_games_by_genre(games, genres) # 오래된 게임 / 최신 게임으로 분리 @@ -70,4 +70,4 @@ def pick_games( olders, newers = _pick_older_newer_games(categorized_games, median_released_at) # 최종 게임 선발 - return _pick_unique_per_category(categorized_games) + return _pick_unique_per_category(olders + newers) From 1d2aa4e04232741c268200c357be2f7332becdca Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:16:09 +0900 Subject: [PATCH 20/31] =?UTF-8?q?test:=20genres=20pick=EC=9D=98=20?= =?UTF-8?q?=EC=95=84=EC=9B=83=ED=92=8B=20=EA=B0=9C=EC=88=98=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=EC=8B=9C=20"=EC=9C=A0=EC=9D=BC=ED=95=9C"=20=EA=B0=9C?= =?UTF-8?q?=EC=88=98=20=EA=B2=80=EC=A6=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/daily_quiz/daily_quiz/test_genre_picker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py index 2616423..086dcd5 100644 --- a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py @@ -5,10 +5,10 @@ @pytest.mark.parametrize("genre_cnt", (5, 10)) -def test_pick_genres는_입력된_개수만큼_장르를_반환해야_한다(genre_cnt: int): +def test_pick_genres는_입력된_개수만큼_유일한_장르_개수를_반환해야_한다(genre_cnt: int): genres = pick_genres(genre_cnt) - assert len(genres) == genre_cnt + assert len(set(genres)) == genre_cnt @pytest.mark.parametrize("setting_genres", (tuple(setting.GAME_GENERES), ("Adventure", "RPG"))) From 99728dea552400e5d6ab6dea813b7d90b5118235 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:16:55 +0900 Subject: [PATCH 21/31] =?UTF-8?q?fix:=20=EC=9E=A5=EB=A5=B4=20random=20?= =?UTF-8?q?=EC=84=A0=ED=83=9D=EC=8B=9C=20=EC=A4=91=EB=B3=B5=EC=84=A0?= =?UTF-8?q?=ED=83=9D=20=EB=B0=A9=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/daily_quiz/genre_picker.py | 2 +- data_scrapers/daily_quiz/daily_quiz/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py index 17cfbc3..cdb3260 100644 --- a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py +++ b/data_scrapers/daily_quiz/daily_quiz/genre_picker.py @@ -4,4 +4,4 @@ def pick_genres(k: int) -> list[str]: - return random.choices(setting.GAME_GENERES, k=k) + return random.sample(setting.GAME_GENERES, k=k) diff --git a/data_scrapers/daily_quiz/daily_quiz/utils.py b/data_scrapers/daily_quiz/daily_quiz/utils.py index 4436964..a97bc40 100644 --- a/data_scrapers/daily_quiz/daily_quiz/utils.py +++ b/data_scrapers/daily_quiz/daily_quiz/utils.py @@ -7,7 +7,7 @@ def divide_randomly(x: Sequence[T], k: int) -> tuple[list[T], list[T]]: idxes = range(len(x)) - a_idxes = random.choices(idxes, k=k) + a_idxes = random.sample(idxes, k=k) b_idxes = list(set(idxes) - set(a_idxes)) a = [x[a_i] for a_i in a_idxes] From 114906553df195b1cc246bf4bee94096c652666c Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:17:15 +0900 Subject: [PATCH 22/31] =?UTF-8?q?test:=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=9D=98=20=EC=9E=A5=EB=A5=B4=20?= =?UTF-8?q?=EC=A4=91=EB=B3=B5=20=EB=AC=B8=EC=A0=9C=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py b/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py index 93dbefa..c87eec5 100644 --- a/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py +++ b/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py @@ -1,4 +1,4 @@ -from random import choices +import random from typing import Sequence from daily_quiz.aws_lambda.model import Game, SaveGameScreenshot, SaveQuiz @@ -11,7 +11,9 @@ class MockLambdaAPI(LambdaAPI): def __init__(self) -> None: self.games = [] for k in range(1, 4): - self.games.extend([create_random_game(genres=choices(setting.GAME_GENERES, k=k)) for i in range(100)]) + self.games.extend( + [create_random_game(genres=random.sample(setting.GAME_GENERES, k=k)) for i in range(100)] + ) self.quizzes = [] def save_quizzes(self, quizzes: Sequence[SaveQuiz]): From 91dc619e1109d0c6280558b537e55581ab5cd573 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:28:19 +0900 Subject: [PATCH 23/31] =?UTF-8?q?test:=20older=20game=20cnt=20=EA=B0=9C?= =?UTF-8?q?=EC=88=98=20=EA=B4=80=EB=A0=A8=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/test_game_picker.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py index 30234f6..9eb8582 100644 --- a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py +++ b/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py @@ -1,3 +1,8 @@ +import random + +import pytest + +from daily_quiz.config import setting from daily_quiz.daily_quiz.game_picker import pick_games from tests.daily_quiz.utils.model import create_random_game @@ -22,3 +27,23 @@ def test_pick_games은_입력한_게임에서_장르별로_1개씩_선택해야_ picked = pick_games(games, genres) assert len(picked) == len(genres) + + +@pytest.mark.parametrize("older_cnt", (0, 2, 5)) +def test_pick_games은_older개수_설정이_변경되어도_적응해야한다(older_cnt: int): + setting.OLDER_GAME_COUNT = older_cnt + + genres = [ + "Action", + "Adventure", + "Massively Multiplayer", + "Strategy", + "RPG", + ] + games = [] + for k in range(1, 4): + games.extend([create_random_game(genres=random.sample(genres, k=k)) for i in range(100)]) + + picked = pick_games(games, genres) + + assert len(picked) == len(genres) From 9d115f646bab40f6c7301474ff83ad3934520dff Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:28:34 +0900 Subject: [PATCH 24/31] =?UTF-8?q?fix:=20=EC=9E=98=EB=AA=BB=EB=90=9C=20?= =?UTF-8?q?=EB=B3=80=EC=88=98=20=EC=82=AC=EC=9A=A9=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_scrapers/daily_quiz/daily_quiz/game_picker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_scrapers/daily_quiz/daily_quiz/game_picker.py b/data_scrapers/daily_quiz/daily_quiz/game_picker.py index 9c5e706..6951bb6 100644 --- a/data_scrapers/daily_quiz/daily_quiz/game_picker.py +++ b/data_scrapers/daily_quiz/daily_quiz/game_picker.py @@ -43,7 +43,7 @@ def _pick_older_newer_games( older_part, newer_part = divide_randomly(categorized_games, setting.OLDER_GAME_COUNT) olders = [_filter_older_games(games, median_released_at) for games in older_part] - newers = [_filter_newer_games(games, median_released_at) for games in older_part] + newers = [_filter_newer_games(games, median_released_at) for games in newer_part] return olders, newers From 265b7834f21f7d489ce85dd9593a27ac76a4df4c Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:39:42 +0900 Subject: [PATCH 25/31] =?UTF-8?q?refactor:=20data=5Fscraper=EC=97=90?= =?UTF-8?q?=EC=84=9C=20aws=5Flambdas=EB=A1=9C=20=EB=94=94=EB=A0=89?= =?UTF-8?q?=ED=84=B0=EB=A6=AC=20=EB=AA=85=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- {data_scrapers => aws_lambdas}/.flake8 | 0 {data_scrapers => aws_lambdas}/.gitignore | 4 +++- .../database_lambda/__init__.py => aws_lambdas/README.md | 0 {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/event.py | 0 .../daily_quiz/aws_lambda/exception.py | 0 .../daily_quiz/aws_lambda/lambda_api.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/model.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/config.py | 0 .../daily_quiz/daily_quiz/game_picker.py | 0 .../daily_quiz/daily_quiz/genre_picker.py | 0 .../daily_quiz/daily_quiz/screenshot_scraper.py | 0 .../daily_quiz/daily_quiz/serivce.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/utils.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/lambda_func.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/logger.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/protocols.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/steam/model.py | 0 {data_scrapers => aws_lambdas}/daily_quiz/steam/steam_api.py | 0 .../daily_quiz/steam/steampowered_api.py | 0 aws_lambdas/database_lambda/__init__.py | 0 {data_scrapers => aws_lambdas}/database_lambda/config.py | 0 {data_scrapers => aws_lambdas}/database_lambda/database.py | 0 {data_scrapers => aws_lambdas}/database_lambda/event.py | 0 {data_scrapers => aws_lambdas}/database_lambda/game/model.py | 0 .../database_lambda/game/repository.py | 0 .../database_lambda/game/service.py | 0 {data_scrapers => aws_lambdas}/database_lambda/genre/model.py | 0 .../database_lambda/genre/repository.py | 0 {data_scrapers => aws_lambdas}/database_lambda/lambda_func.py | 0 {data_scrapers => aws_lambdas}/database_lambda/logger.py | 0 {data_scrapers => aws_lambdas}/database_lambda/model.py | 0 .../database_lambda/quiz/exception.py | 0 {data_scrapers => aws_lambdas}/database_lambda/quiz/model.py | 0 .../database_lambda/quiz/model_factory.py | 0 {data_scrapers => aws_lambdas}/database_lambda/quiz/schema.py | 0 .../database_lambda/quiz/service.py | 0 .../database_lambda/screenshot/model.py | 0 .../database_lambda/screenshot/model_factory.py | 0 .../database_lambda/screenshot/repository.py | 0 .../database_lambda/screenshot/schema.py | 0 .../database_lambda/screenshot/service.py | 0 .../game_updater/aws_lambda/event.py | 0 .../game_updater/aws_lambda/exception.py | 0 .../game_updater/aws_lambda/lambda_api.py | 0 .../game_updater/aws_lambda/model.py | 0 {data_scrapers => aws_lambdas}/game_updater/config.py | 0 {data_scrapers => aws_lambdas}/game_updater/lambda_func.py | 0 {data_scrapers => aws_lambdas}/game_updater/logger.py | 0 {data_scrapers => aws_lambdas}/game_updater/protocols.py | 0 {data_scrapers => aws_lambdas}/game_updater/scraper/game.py | 0 {data_scrapers => aws_lambdas}/game_updater/scraper/model.py | 0 .../game_updater/scraper/service.py | 0 .../game_updater/steam/exception.py | 0 .../game_updater/steam/gamalytic_api.py | 0 {data_scrapers => aws_lambdas}/game_updater/steam/model.py | 0 .../game_updater/steam/steam_api.py | 0 .../game_updater/steam/steampowered_api.py | 0 {data_scrapers => aws_lambdas}/poetry.lock | 0 {data_scrapers => aws_lambdas}/pyproject.toml | 0 {data_scrapers => aws_lambdas}/pytest.ini | 0 .../scripts/build_database_lambda.sh | 0 .../scripts/build_game_updater_lambda.sh | 0 {data_scrapers => aws_lambdas}/scripts/filter_games.py | 0 {data_scrapers => aws_lambdas}/scripts/reset_database.py | 0 {data_scrapers => aws_lambdas}/scripts/save_games.py | 0 {data_scrapers => aws_lambdas}/scripts/scrap_steam_games.py | 0 .../tests/daily_quiz/daily_quiz/test_game_picker.py | 0 .../tests/daily_quiz/daily_quiz/test_genre_picker.py | 0 .../tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py | 0 .../tests/daily_quiz/utils/mock_lambda_api.py | 0 .../tests/daily_quiz/utils/mock_steam_api.py | 0 .../tests/daily_quiz/utils/model.py | 0 .../tests/daily_quiz/utils/steam.py | 0 .../tests/daily_quiz/utils/utils.py | 0 .../tests/database_lambda/conftest.py | 0 .../tests/database_lambda/database.py | 0 .../tests/database_lambda/game/test_game_service.py | 0 .../tests/database_lambda/quiz/test_quiz_service.py | 0 .../database_lambda/screenshot/test_screenshot_service.py | 0 .../tests/database_lambda/utils/game.py | 0 .../tests/database_lambda/utils/genre.py | 0 .../tests/database_lambda/utils/screenshot.py | 0 .../tests/database_lambda/utils/utils.py | 0 .../tests/game_updater/scraper/test_game_scraper.py | 0 .../tests/game_updater/steam/test_steam_api.py | 0 .../tests/game_updater/utils/mock_lambda_api.py | 0 .../tests/game_updater/utils/mock_steam_api.py | 0 .../tests/game_updater/utils/model.py | 0 .../tests/game_updater/utils/steam.py | 0 .../tests/game_updater/utils/utils.py | 0 data_scrapers/test.env | 1 - 91 files changed, 3 insertions(+), 2 deletions(-) rename {data_scrapers => aws_lambdas}/.flake8 (100%) rename {data_scrapers => aws_lambdas}/.gitignore (81%) rename data_scrapers/database_lambda/__init__.py => aws_lambdas/README.md (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/event.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/exception.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/lambda_api.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/aws_lambda/model.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/config.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/game_picker.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/genre_picker.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/screenshot_scraper.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/serivce.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/daily_quiz/utils.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/lambda_func.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/logger.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/protocols.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/steam/model.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/steam/steam_api.py (100%) rename {data_scrapers => aws_lambdas}/daily_quiz/steam/steampowered_api.py (100%) create mode 100644 aws_lambdas/database_lambda/__init__.py rename {data_scrapers => aws_lambdas}/database_lambda/config.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/database.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/event.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/game/model.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/game/repository.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/game/service.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/genre/model.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/genre/repository.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/lambda_func.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/logger.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/model.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/quiz/exception.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/quiz/model.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/quiz/model_factory.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/quiz/schema.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/quiz/service.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/screenshot/model.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/screenshot/model_factory.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/screenshot/repository.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/screenshot/schema.py (100%) rename {data_scrapers => aws_lambdas}/database_lambda/screenshot/service.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/aws_lambda/event.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/aws_lambda/exception.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/aws_lambda/lambda_api.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/aws_lambda/model.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/config.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/lambda_func.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/logger.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/protocols.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/scraper/game.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/scraper/model.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/scraper/service.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/steam/exception.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/steam/gamalytic_api.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/steam/model.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/steam/steam_api.py (100%) rename {data_scrapers => aws_lambdas}/game_updater/steam/steampowered_api.py (100%) rename {data_scrapers => aws_lambdas}/poetry.lock (100%) rename {data_scrapers => aws_lambdas}/pyproject.toml (100%) rename {data_scrapers => aws_lambdas}/pytest.ini (100%) rename {data_scrapers => aws_lambdas}/scripts/build_database_lambda.sh (100%) rename {data_scrapers => aws_lambdas}/scripts/build_game_updater_lambda.sh (100%) rename {data_scrapers => aws_lambdas}/scripts/filter_games.py (100%) rename {data_scrapers => aws_lambdas}/scripts/reset_database.py (100%) rename {data_scrapers => aws_lambdas}/scripts/save_games.py (100%) rename {data_scrapers => aws_lambdas}/scripts/scrap_steam_games.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/daily_quiz/test_game_picker.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/daily_quiz/test_genre_picker.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/utils/mock_lambda_api.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/utils/mock_steam_api.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/utils/model.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/utils/steam.py (100%) rename {data_scrapers => aws_lambdas}/tests/daily_quiz/utils/utils.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/conftest.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/database.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/game/test_game_service.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/quiz/test_quiz_service.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/screenshot/test_screenshot_service.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/utils/game.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/utils/genre.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/utils/screenshot.py (100%) rename {data_scrapers => aws_lambdas}/tests/database_lambda/utils/utils.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/scraper/test_game_scraper.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/steam/test_steam_api.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/utils/mock_lambda_api.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/utils/mock_steam_api.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/utils/model.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/utils/steam.py (100%) rename {data_scrapers => aws_lambdas}/tests/game_updater/utils/utils.py (100%) delete mode 100644 data_scrapers/test.env diff --git a/data_scrapers/.flake8 b/aws_lambdas/.flake8 similarity index 100% rename from data_scrapers/.flake8 rename to aws_lambdas/.flake8 diff --git a/data_scrapers/.gitignore b/aws_lambdas/.gitignore similarity index 81% rename from data_scrapers/.gitignore rename to aws_lambdas/.gitignore index 434462f..56a833c 100644 --- a/data_scrapers/.gitignore +++ b/aws_lambdas/.gitignore @@ -12,4 +12,6 @@ requirements.txt # vscode .vscode # data -*.csv \ No newline at end of file +*.csv +# jupyter notebook +.ipynb \ No newline at end of file diff --git a/data_scrapers/database_lambda/__init__.py b/aws_lambdas/README.md similarity index 100% rename from data_scrapers/database_lambda/__init__.py rename to aws_lambdas/README.md diff --git a/data_scrapers/daily_quiz/aws_lambda/event.py b/aws_lambdas/daily_quiz/aws_lambda/event.py similarity index 100% rename from data_scrapers/daily_quiz/aws_lambda/event.py rename to aws_lambdas/daily_quiz/aws_lambda/event.py diff --git a/data_scrapers/daily_quiz/aws_lambda/exception.py b/aws_lambdas/daily_quiz/aws_lambda/exception.py similarity index 100% rename from data_scrapers/daily_quiz/aws_lambda/exception.py rename to aws_lambdas/daily_quiz/aws_lambda/exception.py diff --git a/data_scrapers/daily_quiz/aws_lambda/lambda_api.py b/aws_lambdas/daily_quiz/aws_lambda/lambda_api.py similarity index 100% rename from data_scrapers/daily_quiz/aws_lambda/lambda_api.py rename to aws_lambdas/daily_quiz/aws_lambda/lambda_api.py diff --git a/data_scrapers/daily_quiz/aws_lambda/model.py b/aws_lambdas/daily_quiz/aws_lambda/model.py similarity index 100% rename from data_scrapers/daily_quiz/aws_lambda/model.py rename to aws_lambdas/daily_quiz/aws_lambda/model.py diff --git a/data_scrapers/daily_quiz/config.py b/aws_lambdas/daily_quiz/config.py similarity index 100% rename from data_scrapers/daily_quiz/config.py rename to aws_lambdas/daily_quiz/config.py diff --git a/data_scrapers/daily_quiz/daily_quiz/game_picker.py b/aws_lambdas/daily_quiz/daily_quiz/game_picker.py similarity index 100% rename from data_scrapers/daily_quiz/daily_quiz/game_picker.py rename to aws_lambdas/daily_quiz/daily_quiz/game_picker.py diff --git a/data_scrapers/daily_quiz/daily_quiz/genre_picker.py b/aws_lambdas/daily_quiz/daily_quiz/genre_picker.py similarity index 100% rename from data_scrapers/daily_quiz/daily_quiz/genre_picker.py rename to aws_lambdas/daily_quiz/daily_quiz/genre_picker.py diff --git a/data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py b/aws_lambdas/daily_quiz/daily_quiz/screenshot_scraper.py similarity index 100% rename from data_scrapers/daily_quiz/daily_quiz/screenshot_scraper.py rename to aws_lambdas/daily_quiz/daily_quiz/screenshot_scraper.py diff --git a/data_scrapers/daily_quiz/daily_quiz/serivce.py b/aws_lambdas/daily_quiz/daily_quiz/serivce.py similarity index 100% rename from data_scrapers/daily_quiz/daily_quiz/serivce.py rename to aws_lambdas/daily_quiz/daily_quiz/serivce.py diff --git a/data_scrapers/daily_quiz/daily_quiz/utils.py b/aws_lambdas/daily_quiz/daily_quiz/utils.py similarity index 100% rename from data_scrapers/daily_quiz/daily_quiz/utils.py rename to aws_lambdas/daily_quiz/daily_quiz/utils.py diff --git a/data_scrapers/daily_quiz/lambda_func.py b/aws_lambdas/daily_quiz/lambda_func.py similarity index 100% rename from data_scrapers/daily_quiz/lambda_func.py rename to aws_lambdas/daily_quiz/lambda_func.py diff --git a/data_scrapers/daily_quiz/logger.py b/aws_lambdas/daily_quiz/logger.py similarity index 100% rename from data_scrapers/daily_quiz/logger.py rename to aws_lambdas/daily_quiz/logger.py diff --git a/data_scrapers/daily_quiz/protocols.py b/aws_lambdas/daily_quiz/protocols.py similarity index 100% rename from data_scrapers/daily_quiz/protocols.py rename to aws_lambdas/daily_quiz/protocols.py diff --git a/data_scrapers/daily_quiz/steam/model.py b/aws_lambdas/daily_quiz/steam/model.py similarity index 100% rename from data_scrapers/daily_quiz/steam/model.py rename to aws_lambdas/daily_quiz/steam/model.py diff --git a/data_scrapers/daily_quiz/steam/steam_api.py b/aws_lambdas/daily_quiz/steam/steam_api.py similarity index 100% rename from data_scrapers/daily_quiz/steam/steam_api.py rename to aws_lambdas/daily_quiz/steam/steam_api.py diff --git a/data_scrapers/daily_quiz/steam/steampowered_api.py b/aws_lambdas/daily_quiz/steam/steampowered_api.py similarity index 100% rename from data_scrapers/daily_quiz/steam/steampowered_api.py rename to aws_lambdas/daily_quiz/steam/steampowered_api.py diff --git a/aws_lambdas/database_lambda/__init__.py b/aws_lambdas/database_lambda/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_scrapers/database_lambda/config.py b/aws_lambdas/database_lambda/config.py similarity index 100% rename from data_scrapers/database_lambda/config.py rename to aws_lambdas/database_lambda/config.py diff --git a/data_scrapers/database_lambda/database.py b/aws_lambdas/database_lambda/database.py similarity index 100% rename from data_scrapers/database_lambda/database.py rename to aws_lambdas/database_lambda/database.py diff --git a/data_scrapers/database_lambda/event.py b/aws_lambdas/database_lambda/event.py similarity index 100% rename from data_scrapers/database_lambda/event.py rename to aws_lambdas/database_lambda/event.py diff --git a/data_scrapers/database_lambda/game/model.py b/aws_lambdas/database_lambda/game/model.py similarity index 100% rename from data_scrapers/database_lambda/game/model.py rename to aws_lambdas/database_lambda/game/model.py diff --git a/data_scrapers/database_lambda/game/repository.py b/aws_lambdas/database_lambda/game/repository.py similarity index 100% rename from data_scrapers/database_lambda/game/repository.py rename to aws_lambdas/database_lambda/game/repository.py diff --git a/data_scrapers/database_lambda/game/service.py b/aws_lambdas/database_lambda/game/service.py similarity index 100% rename from data_scrapers/database_lambda/game/service.py rename to aws_lambdas/database_lambda/game/service.py diff --git a/data_scrapers/database_lambda/genre/model.py b/aws_lambdas/database_lambda/genre/model.py similarity index 100% rename from data_scrapers/database_lambda/genre/model.py rename to aws_lambdas/database_lambda/genre/model.py diff --git a/data_scrapers/database_lambda/genre/repository.py b/aws_lambdas/database_lambda/genre/repository.py similarity index 100% rename from data_scrapers/database_lambda/genre/repository.py rename to aws_lambdas/database_lambda/genre/repository.py diff --git a/data_scrapers/database_lambda/lambda_func.py b/aws_lambdas/database_lambda/lambda_func.py similarity index 100% rename from data_scrapers/database_lambda/lambda_func.py rename to aws_lambdas/database_lambda/lambda_func.py diff --git a/data_scrapers/database_lambda/logger.py b/aws_lambdas/database_lambda/logger.py similarity index 100% rename from data_scrapers/database_lambda/logger.py rename to aws_lambdas/database_lambda/logger.py diff --git a/data_scrapers/database_lambda/model.py b/aws_lambdas/database_lambda/model.py similarity index 100% rename from data_scrapers/database_lambda/model.py rename to aws_lambdas/database_lambda/model.py diff --git a/data_scrapers/database_lambda/quiz/exception.py b/aws_lambdas/database_lambda/quiz/exception.py similarity index 100% rename from data_scrapers/database_lambda/quiz/exception.py rename to aws_lambdas/database_lambda/quiz/exception.py diff --git a/data_scrapers/database_lambda/quiz/model.py b/aws_lambdas/database_lambda/quiz/model.py similarity index 100% rename from data_scrapers/database_lambda/quiz/model.py rename to aws_lambdas/database_lambda/quiz/model.py diff --git a/data_scrapers/database_lambda/quiz/model_factory.py b/aws_lambdas/database_lambda/quiz/model_factory.py similarity index 100% rename from data_scrapers/database_lambda/quiz/model_factory.py rename to aws_lambdas/database_lambda/quiz/model_factory.py diff --git a/data_scrapers/database_lambda/quiz/schema.py b/aws_lambdas/database_lambda/quiz/schema.py similarity index 100% rename from data_scrapers/database_lambda/quiz/schema.py rename to aws_lambdas/database_lambda/quiz/schema.py diff --git a/data_scrapers/database_lambda/quiz/service.py b/aws_lambdas/database_lambda/quiz/service.py similarity index 100% rename from data_scrapers/database_lambda/quiz/service.py rename to aws_lambdas/database_lambda/quiz/service.py diff --git a/data_scrapers/database_lambda/screenshot/model.py b/aws_lambdas/database_lambda/screenshot/model.py similarity index 100% rename from data_scrapers/database_lambda/screenshot/model.py rename to aws_lambdas/database_lambda/screenshot/model.py diff --git a/data_scrapers/database_lambda/screenshot/model_factory.py b/aws_lambdas/database_lambda/screenshot/model_factory.py similarity index 100% rename from data_scrapers/database_lambda/screenshot/model_factory.py rename to aws_lambdas/database_lambda/screenshot/model_factory.py diff --git a/data_scrapers/database_lambda/screenshot/repository.py b/aws_lambdas/database_lambda/screenshot/repository.py similarity index 100% rename from data_scrapers/database_lambda/screenshot/repository.py rename to aws_lambdas/database_lambda/screenshot/repository.py diff --git a/data_scrapers/database_lambda/screenshot/schema.py b/aws_lambdas/database_lambda/screenshot/schema.py similarity index 100% rename from data_scrapers/database_lambda/screenshot/schema.py rename to aws_lambdas/database_lambda/screenshot/schema.py diff --git a/data_scrapers/database_lambda/screenshot/service.py b/aws_lambdas/database_lambda/screenshot/service.py similarity index 100% rename from data_scrapers/database_lambda/screenshot/service.py rename to aws_lambdas/database_lambda/screenshot/service.py diff --git a/data_scrapers/game_updater/aws_lambda/event.py b/aws_lambdas/game_updater/aws_lambda/event.py similarity index 100% rename from data_scrapers/game_updater/aws_lambda/event.py rename to aws_lambdas/game_updater/aws_lambda/event.py diff --git a/data_scrapers/game_updater/aws_lambda/exception.py b/aws_lambdas/game_updater/aws_lambda/exception.py similarity index 100% rename from data_scrapers/game_updater/aws_lambda/exception.py rename to aws_lambdas/game_updater/aws_lambda/exception.py diff --git a/data_scrapers/game_updater/aws_lambda/lambda_api.py b/aws_lambdas/game_updater/aws_lambda/lambda_api.py similarity index 100% rename from data_scrapers/game_updater/aws_lambda/lambda_api.py rename to aws_lambdas/game_updater/aws_lambda/lambda_api.py diff --git a/data_scrapers/game_updater/aws_lambda/model.py b/aws_lambdas/game_updater/aws_lambda/model.py similarity index 100% rename from data_scrapers/game_updater/aws_lambda/model.py rename to aws_lambdas/game_updater/aws_lambda/model.py diff --git a/data_scrapers/game_updater/config.py b/aws_lambdas/game_updater/config.py similarity index 100% rename from data_scrapers/game_updater/config.py rename to aws_lambdas/game_updater/config.py diff --git a/data_scrapers/game_updater/lambda_func.py b/aws_lambdas/game_updater/lambda_func.py similarity index 100% rename from data_scrapers/game_updater/lambda_func.py rename to aws_lambdas/game_updater/lambda_func.py diff --git a/data_scrapers/game_updater/logger.py b/aws_lambdas/game_updater/logger.py similarity index 100% rename from data_scrapers/game_updater/logger.py rename to aws_lambdas/game_updater/logger.py diff --git a/data_scrapers/game_updater/protocols.py b/aws_lambdas/game_updater/protocols.py similarity index 100% rename from data_scrapers/game_updater/protocols.py rename to aws_lambdas/game_updater/protocols.py diff --git a/data_scrapers/game_updater/scraper/game.py b/aws_lambdas/game_updater/scraper/game.py similarity index 100% rename from data_scrapers/game_updater/scraper/game.py rename to aws_lambdas/game_updater/scraper/game.py diff --git a/data_scrapers/game_updater/scraper/model.py b/aws_lambdas/game_updater/scraper/model.py similarity index 100% rename from data_scrapers/game_updater/scraper/model.py rename to aws_lambdas/game_updater/scraper/model.py diff --git a/data_scrapers/game_updater/scraper/service.py b/aws_lambdas/game_updater/scraper/service.py similarity index 100% rename from data_scrapers/game_updater/scraper/service.py rename to aws_lambdas/game_updater/scraper/service.py diff --git a/data_scrapers/game_updater/steam/exception.py b/aws_lambdas/game_updater/steam/exception.py similarity index 100% rename from data_scrapers/game_updater/steam/exception.py rename to aws_lambdas/game_updater/steam/exception.py diff --git a/data_scrapers/game_updater/steam/gamalytic_api.py b/aws_lambdas/game_updater/steam/gamalytic_api.py similarity index 100% rename from data_scrapers/game_updater/steam/gamalytic_api.py rename to aws_lambdas/game_updater/steam/gamalytic_api.py diff --git a/data_scrapers/game_updater/steam/model.py b/aws_lambdas/game_updater/steam/model.py similarity index 100% rename from data_scrapers/game_updater/steam/model.py rename to aws_lambdas/game_updater/steam/model.py diff --git a/data_scrapers/game_updater/steam/steam_api.py b/aws_lambdas/game_updater/steam/steam_api.py similarity index 100% rename from data_scrapers/game_updater/steam/steam_api.py rename to aws_lambdas/game_updater/steam/steam_api.py diff --git a/data_scrapers/game_updater/steam/steampowered_api.py b/aws_lambdas/game_updater/steam/steampowered_api.py similarity index 100% rename from data_scrapers/game_updater/steam/steampowered_api.py rename to aws_lambdas/game_updater/steam/steampowered_api.py diff --git a/data_scrapers/poetry.lock b/aws_lambdas/poetry.lock similarity index 100% rename from data_scrapers/poetry.lock rename to aws_lambdas/poetry.lock diff --git a/data_scrapers/pyproject.toml b/aws_lambdas/pyproject.toml similarity index 100% rename from data_scrapers/pyproject.toml rename to aws_lambdas/pyproject.toml diff --git a/data_scrapers/pytest.ini b/aws_lambdas/pytest.ini similarity index 100% rename from data_scrapers/pytest.ini rename to aws_lambdas/pytest.ini diff --git a/data_scrapers/scripts/build_database_lambda.sh b/aws_lambdas/scripts/build_database_lambda.sh similarity index 100% rename from data_scrapers/scripts/build_database_lambda.sh rename to aws_lambdas/scripts/build_database_lambda.sh diff --git a/data_scrapers/scripts/build_game_updater_lambda.sh b/aws_lambdas/scripts/build_game_updater_lambda.sh similarity index 100% rename from data_scrapers/scripts/build_game_updater_lambda.sh rename to aws_lambdas/scripts/build_game_updater_lambda.sh diff --git a/data_scrapers/scripts/filter_games.py b/aws_lambdas/scripts/filter_games.py similarity index 100% rename from data_scrapers/scripts/filter_games.py rename to aws_lambdas/scripts/filter_games.py diff --git a/data_scrapers/scripts/reset_database.py b/aws_lambdas/scripts/reset_database.py similarity index 100% rename from data_scrapers/scripts/reset_database.py rename to aws_lambdas/scripts/reset_database.py diff --git a/data_scrapers/scripts/save_games.py b/aws_lambdas/scripts/save_games.py similarity index 100% rename from data_scrapers/scripts/save_games.py rename to aws_lambdas/scripts/save_games.py diff --git a/data_scrapers/scripts/scrap_steam_games.py b/aws_lambdas/scripts/scrap_steam_games.py similarity index 100% rename from data_scrapers/scripts/scrap_steam_games.py rename to aws_lambdas/scripts/scrap_steam_games.py diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py b/aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py similarity index 100% rename from data_scrapers/tests/daily_quiz/daily_quiz/test_game_picker.py rename to aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py b/aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py similarity index 100% rename from data_scrapers/tests/daily_quiz/daily_quiz/test_genre_picker.py rename to aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py diff --git a/data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py b/aws_lambdas/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py similarity index 100% rename from data_scrapers/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py rename to aws_lambdas/tests/daily_quiz/daily_quiz/test_new_daily_quizzes.py diff --git a/data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py b/aws_lambdas/tests/daily_quiz/utils/mock_lambda_api.py similarity index 100% rename from data_scrapers/tests/daily_quiz/utils/mock_lambda_api.py rename to aws_lambdas/tests/daily_quiz/utils/mock_lambda_api.py diff --git a/data_scrapers/tests/daily_quiz/utils/mock_steam_api.py b/aws_lambdas/tests/daily_quiz/utils/mock_steam_api.py similarity index 100% rename from data_scrapers/tests/daily_quiz/utils/mock_steam_api.py rename to aws_lambdas/tests/daily_quiz/utils/mock_steam_api.py diff --git a/data_scrapers/tests/daily_quiz/utils/model.py b/aws_lambdas/tests/daily_quiz/utils/model.py similarity index 100% rename from data_scrapers/tests/daily_quiz/utils/model.py rename to aws_lambdas/tests/daily_quiz/utils/model.py diff --git a/data_scrapers/tests/daily_quiz/utils/steam.py b/aws_lambdas/tests/daily_quiz/utils/steam.py similarity index 100% rename from data_scrapers/tests/daily_quiz/utils/steam.py rename to aws_lambdas/tests/daily_quiz/utils/steam.py diff --git a/data_scrapers/tests/daily_quiz/utils/utils.py b/aws_lambdas/tests/daily_quiz/utils/utils.py similarity index 100% rename from data_scrapers/tests/daily_quiz/utils/utils.py rename to aws_lambdas/tests/daily_quiz/utils/utils.py diff --git a/data_scrapers/tests/database_lambda/conftest.py b/aws_lambdas/tests/database_lambda/conftest.py similarity index 100% rename from data_scrapers/tests/database_lambda/conftest.py rename to aws_lambdas/tests/database_lambda/conftest.py diff --git a/data_scrapers/tests/database_lambda/database.py b/aws_lambdas/tests/database_lambda/database.py similarity index 100% rename from data_scrapers/tests/database_lambda/database.py rename to aws_lambdas/tests/database_lambda/database.py diff --git a/data_scrapers/tests/database_lambda/game/test_game_service.py b/aws_lambdas/tests/database_lambda/game/test_game_service.py similarity index 100% rename from data_scrapers/tests/database_lambda/game/test_game_service.py rename to aws_lambdas/tests/database_lambda/game/test_game_service.py diff --git a/data_scrapers/tests/database_lambda/quiz/test_quiz_service.py b/aws_lambdas/tests/database_lambda/quiz/test_quiz_service.py similarity index 100% rename from data_scrapers/tests/database_lambda/quiz/test_quiz_service.py rename to aws_lambdas/tests/database_lambda/quiz/test_quiz_service.py diff --git a/data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py b/aws_lambdas/tests/database_lambda/screenshot/test_screenshot_service.py similarity index 100% rename from data_scrapers/tests/database_lambda/screenshot/test_screenshot_service.py rename to aws_lambdas/tests/database_lambda/screenshot/test_screenshot_service.py diff --git a/data_scrapers/tests/database_lambda/utils/game.py b/aws_lambdas/tests/database_lambda/utils/game.py similarity index 100% rename from data_scrapers/tests/database_lambda/utils/game.py rename to aws_lambdas/tests/database_lambda/utils/game.py diff --git a/data_scrapers/tests/database_lambda/utils/genre.py b/aws_lambdas/tests/database_lambda/utils/genre.py similarity index 100% rename from data_scrapers/tests/database_lambda/utils/genre.py rename to aws_lambdas/tests/database_lambda/utils/genre.py diff --git a/data_scrapers/tests/database_lambda/utils/screenshot.py b/aws_lambdas/tests/database_lambda/utils/screenshot.py similarity index 100% rename from data_scrapers/tests/database_lambda/utils/screenshot.py rename to aws_lambdas/tests/database_lambda/utils/screenshot.py diff --git a/data_scrapers/tests/database_lambda/utils/utils.py b/aws_lambdas/tests/database_lambda/utils/utils.py similarity index 100% rename from data_scrapers/tests/database_lambda/utils/utils.py rename to aws_lambdas/tests/database_lambda/utils/utils.py diff --git a/data_scrapers/tests/game_updater/scraper/test_game_scraper.py b/aws_lambdas/tests/game_updater/scraper/test_game_scraper.py similarity index 100% rename from data_scrapers/tests/game_updater/scraper/test_game_scraper.py rename to aws_lambdas/tests/game_updater/scraper/test_game_scraper.py diff --git a/data_scrapers/tests/game_updater/steam/test_steam_api.py b/aws_lambdas/tests/game_updater/steam/test_steam_api.py similarity index 100% rename from data_scrapers/tests/game_updater/steam/test_steam_api.py rename to aws_lambdas/tests/game_updater/steam/test_steam_api.py diff --git a/data_scrapers/tests/game_updater/utils/mock_lambda_api.py b/aws_lambdas/tests/game_updater/utils/mock_lambda_api.py similarity index 100% rename from data_scrapers/tests/game_updater/utils/mock_lambda_api.py rename to aws_lambdas/tests/game_updater/utils/mock_lambda_api.py diff --git a/data_scrapers/tests/game_updater/utils/mock_steam_api.py b/aws_lambdas/tests/game_updater/utils/mock_steam_api.py similarity index 100% rename from data_scrapers/tests/game_updater/utils/mock_steam_api.py rename to aws_lambdas/tests/game_updater/utils/mock_steam_api.py diff --git a/data_scrapers/tests/game_updater/utils/model.py b/aws_lambdas/tests/game_updater/utils/model.py similarity index 100% rename from data_scrapers/tests/game_updater/utils/model.py rename to aws_lambdas/tests/game_updater/utils/model.py diff --git a/data_scrapers/tests/game_updater/utils/steam.py b/aws_lambdas/tests/game_updater/utils/steam.py similarity index 100% rename from data_scrapers/tests/game_updater/utils/steam.py rename to aws_lambdas/tests/game_updater/utils/steam.py diff --git a/data_scrapers/tests/game_updater/utils/utils.py b/aws_lambdas/tests/game_updater/utils/utils.py similarity index 100% rename from data_scrapers/tests/game_updater/utils/utils.py rename to aws_lambdas/tests/game_updater/utils/utils.py diff --git a/data_scrapers/test.env b/data_scrapers/test.env deleted file mode 100644 index c6f52e2..0000000 --- a/data_scrapers/test.env +++ /dev/null @@ -1 +0,0 @@ -DATABASE_URL=sqlite:// \ No newline at end of file From a1198d6d5d919f1519da8c972a857011f19d7ef7 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:41:43 +0900 Subject: [PATCH 26/31] =?UTF-8?q?refactor:=20aws=20lambda=20=EA=B9=83?= =?UTF-8?q?=ED=97=88=EB=B8=8C=20=EC=97=91=EC=85=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/{test-scraper.yaml => test-lambda.yaml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{test-scraper.yaml => test-lambda.yaml} (94%) diff --git a/.github/workflows/test-scraper.yaml b/.github/workflows/test-lambda.yaml similarity index 94% rename from .github/workflows/test-scraper.yaml rename to .github/workflows/test-lambda.yaml index 85be0bd..fc28619 100644 --- a/.github/workflows/test-scraper.yaml +++ b/.github/workflows/test-lambda.yaml @@ -1,4 +1,4 @@ -name: Test scraper +name: Test aws lambda on: pull_request: @@ -10,7 +10,7 @@ jobs: test: runs-on: ubuntu-latest env: - working-directory: ./data_scrapers + working-directory: ./aws_lambdas steps: - uses: actions/checkout@v3 From 50a3536d2e2f8aed0d6c53d027386ec615cceef0 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Wed, 7 Feb 2024 23:48:12 +0900 Subject: [PATCH 27/31] =?UTF-8?q?test:=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20?= =?UTF-8?q?=ED=99=98=EA=B2=BD=EB=B3=80=EC=88=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- aws_lambdas/test.env | 1 + 1 file changed, 1 insertion(+) create mode 100644 aws_lambdas/test.env diff --git a/aws_lambdas/test.env b/aws_lambdas/test.env new file mode 100644 index 0000000..c6f52e2 --- /dev/null +++ b/aws_lambdas/test.env @@ -0,0 +1 @@ +DATABASE_URL=sqlite:// \ No newline at end of file From 0719f03bbbd41bac05f4f0ad1cb4901b701696e6 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Thu, 8 Feb 2024 04:37:15 +0900 Subject: [PATCH 28/31] =?UTF-8?q?test:=20pick=5Fgames=EC=97=90=20=EB=8C=80?= =?UTF-8?q?=ED=95=9C=20=EC=98=88=EC=99=B8=20=EC=BC=80=EC=9D=B4=EC=8A=A4=20?= =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/exception.py | 2 + .../daily_quiz/daily_quiz/test_game_picker.py | 102 ++++++++++++++---- 2 files changed, 84 insertions(+), 20 deletions(-) create mode 100644 aws_lambdas/daily_quiz/daily_quiz/exception.py diff --git a/aws_lambdas/daily_quiz/daily_quiz/exception.py b/aws_lambdas/daily_quiz/daily_quiz/exception.py new file mode 100644 index 0000000..6b4df84 --- /dev/null +++ b/aws_lambdas/daily_quiz/daily_quiz/exception.py @@ -0,0 +1,2 @@ +class NotEnoughGamesError(Exception): + pass diff --git a/aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py b/aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py index 9eb8582..ac726d5 100644 --- a/aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py +++ b/aws_lambdas/tests/daily_quiz/daily_quiz/test_game_picker.py @@ -1,27 +1,55 @@ +import datetime import random import pytest +from daily_quiz.aws_lambda.model import Game from daily_quiz.config import setting +from daily_quiz.daily_quiz.exception import NotEnoughGamesError from daily_quiz.daily_quiz.game_picker import pick_games from tests.daily_quiz.utils.model import create_random_game +OLDER_RELEASED_AT = datetime.datetime(year=1990, month=1, day=14) +NEWER_RELEASED_AT = datetime.datetime(year=2025, month=1, day=14) +DEFAULT_GENRES = [ + "Action", + "Adventure", + "Massively Multiplayer", + "Strategy", + "RPG", +] + + +def random_older_games(genres: list[str]) -> list[Game]: + games = [] + for day in range(1, 3): + older_released_at = datetime.datetime(year=1990, month=1, day=day) + games.append(create_random_game(genres=genres, released_at=older_released_at)) + + return games + + +def random_newer_games(genres: list[str]) -> list[Game]: + games = [] + for day in range(1, 3): + newer_released_at = datetime.datetime(year=2025, month=1, day=day) + games.append(create_random_game(genres=genres, released_at=newer_released_at)) + + return games + + +def random_older_newer_games(genres: list[str]) -> list[Game]: + return [*random_older_games(genres), *random_newer_games(genres)] + def test_pick_games은_입력한_게임에서_장르별로_1개씩_선택해야_한다(): - genres = [ - "Action", - "Adventure", - "Massively Multiplayer", - "Strategy", - "RPG", - ] + genres = DEFAULT_GENRES games = [ - create_random_game(genres=["Action", "Adventure"]), - create_random_game(genres=["Action", "Massively Multiplayer"]), - create_random_game(genres=["Adventure", "Massively Multiplayer"]), - create_random_game(genres=["Adventure", "Strategy"]), - create_random_game(genres=["Massively Multiplayer", "Strategy"]), - create_random_game(genres=["Strategy", "RPG"]), + *random_older_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_older_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy"]), + *random_older_newer_games(genres=["Action", "Adventure", "Massively Multiplayer"]), + *random_older_newer_games(genres=["Action", "Adventure"]), + *random_older_newer_games(genres=["Action"]), ] picked = pick_games(games, genres) @@ -29,17 +57,51 @@ def test_pick_games은_입력한_게임에서_장르별로_1개씩_선택해야_ assert len(picked) == len(genres) +def test_pick_games은_장르별_유일한_게임을_선택할_수_없을경우_예외를_던져야한다(): + genres = DEFAULT_GENRES + games = [ + *random_older_newer_games(genres=["Action", "Adventure"]), + *random_older_newer_games(genres=["Action", "Adventure"]), + *random_older_newer_games(genres=["Action", "Adventure"]), + *random_older_newer_games(genres=["Action", "Adventure"]), + *random_older_newer_games(genres=["Action", "Adventure"]), + ] + + with pytest.raises(NotEnoughGamesError): + pick_games(games, genres) + + +def test_pick_games은_오래된_혹은_신규_게임을_선택할_수_없는경우_예외가_발생할_수_있다(): + # 예외가 항상 발생하는 극단적인 상황에 대해서만 다룹니다. + genres = DEFAULT_GENRES + games = [ + *random_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_newer_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + ] + + with pytest.raises(NotEnoughGamesError): + pick_games(games, genres) + + games = [ + *random_older_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_older_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_older_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_older_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + *random_older_games(genres=["Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"]), + ] + + with pytest.raises(NotEnoughGamesError): + pick_games(games, genres) + + @pytest.mark.parametrize("older_cnt", (0, 2, 5)) def test_pick_games은_older개수_설정이_변경되어도_적응해야한다(older_cnt: int): setting.OLDER_GAME_COUNT = older_cnt - genres = [ - "Action", - "Adventure", - "Massively Multiplayer", - "Strategy", - "RPG", - ] + genres = DEFAULT_GENRES games = [] for k in range(1, 4): games.extend([create_random_game(genres=random.sample(genres, k=k)) for i in range(100)]) From da0e9a3dd63631478e4270ac8cb4e286737dc5eb Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Thu, 8 Feb 2024 04:47:26 +0900 Subject: [PATCH 29/31] =?UTF-8?q?feat:=20=EA=B2=8C=EC=9E=84=20=EA=B0=9C?= =?UTF-8?q?=EC=88=98=20=EA=B2=80=EC=A6=9D=EA=B8=B0=EB=8A=A5=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/daily_quiz/game_picker.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/aws_lambdas/daily_quiz/daily_quiz/game_picker.py b/aws_lambdas/daily_quiz/daily_quiz/game_picker.py index 6951bb6..b3dd141 100644 --- a/aws_lambdas/daily_quiz/daily_quiz/game_picker.py +++ b/aws_lambdas/daily_quiz/daily_quiz/game_picker.py @@ -1,10 +1,11 @@ import random from collections import defaultdict +from collections.abc import Collection, Iterable, Sequence from datetime import datetime -from typing import Iterable, Sequence from ..aws_lambda.model import Game from ..config import setting +from .exception import NotEnoughGamesError from .utils import divide_randomly GameGroup = list[Game] @@ -53,15 +54,29 @@ def _pick_unique_per_category(categorized_games: Iterable[GameGroup]) -> set[Gam for games in sorted(categorized_games, key=len): games_ = list(set(games) - unique_games) + + if len(games_) == 0: + flat = [g for g in games for games in categorized_games] + raise NotEnoughGamesError( + f"게임의 수가 너무 적어 게임 선택 알고리즘을 작동할 수 없습니다. 지금까지 선발된 게임: {set(flat)}" + ) + game = random.choice(games_) unique_games.add(game) return unique_games +def _validate_final_games(games: Collection, genres: Collection): + if len(games) != len(genres): + raise NotEnoughGamesError( + f"게임의 수가 너무 적어 게임 선택 알고리즘을 작동할 수 없습니다. 최종 선발된 게임: {games}" + ) + + def pick_games( games: Iterable[Game], - genres: Iterable[str], + genres: Sequence[str], ) -> set[Game]: categorized_games = _categorize_games_by_genre(games, genres) @@ -70,4 +85,6 @@ def pick_games( olders, newers = _pick_older_newer_games(categorized_games, median_released_at) # 최종 게임 선발 - return _pick_unique_per_category(olders + newers) + final_games = _pick_unique_per_category(olders + newers) + _validate_final_games(final_games, genres) + return final_games From bd0f849ef7ed40eed9a52cbb5267c3bcb8266ebb Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Thu, 8 Feb 2024 04:50:26 +0900 Subject: [PATCH 30/31] =?UTF-8?q?test:=20=EC=9E=98=EB=AA=BB=EB=90=9C=20?= =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=BC=80=EC=9D=B4=EC=8A=A4=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily_quiz/test_genre_picker.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py b/aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py index 086dcd5..1d881be 100644 --- a/aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py +++ b/aws_lambdas/tests/daily_quiz/daily_quiz/test_genre_picker.py @@ -11,8 +11,25 @@ def test_pick_genres는_입력된_개수만큼_유일한_장르_개수를_반환 assert len(set(genres)) == genre_cnt -@pytest.mark.parametrize("setting_genres", (tuple(setting.GAME_GENERES), ("Adventure", "RPG"))) -@pytest.mark.parametrize("genre_cnt", (5, 10)) +@pytest.mark.parametrize( + "setting_genres", + ( + ("Action", "Adventure", "Massively Multiplayer", "Strategy", "RPG"), + ( + "Action", + "Adventure", + "Massively Multiplayer", + "Strategy", + "RPG", + "Indie", + "Simulation", + "Casual", + "Racing", + "Sports", + ), + ), +) +@pytest.mark.parametrize("genre_cnt", (1, 5)) def test_pick_genres는_설정값에서_장르를_선택해야_한다(genre_cnt: int, setting_genres: tuple[str]): setting.GAME_GENERES = setting_genres genres = pick_genres(genre_cnt) From 3ef4ef065bd3a8669de824a3f41f26ab888ecb24 Mon Sep 17 00:00:00 2001 From: 2jun0 Date: Thu, 8 Feb 2024 04:52:47 +0900 Subject: [PATCH 31/31] =?UTF-8?q?test:=20=EC=99=B8=EB=B6=80=20api=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=A3=BC?= =?UTF-8?q?=EC=84=9D=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../game_updater/steam/test_steam_api.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/aws_lambdas/tests/game_updater/steam/test_steam_api.py b/aws_lambdas/tests/game_updater/steam/test_steam_api.py index 7e50bb8..0c3af4e 100644 --- a/aws_lambdas/tests/game_updater/steam/test_steam_api.py +++ b/aws_lambdas/tests/game_updater/steam/test_steam_api.py @@ -1,44 +1,44 @@ -import pytest +# import pytest -from game_updater.steam.exception import SteamAPINoContentsException -from game_updater.steam.steam_api import SteamAPI +# from game_updater.steam.exception import SteamAPINoContentsException +# from game_updater.steam.steam_api import SteamAPI -@pytest.fixture -def app_id() -> int: - return 70 # half-life +# @pytest.fixture +# def app_id() -> int: +# return 70 # half-life -@pytest.fixture -def no_detail_app_id() -> int: - return 1599340 # Lost Ark +# @pytest.fixture +# def no_detail_app_id() -> int: +# return 1599340 # Lost Ark -@pytest.fixture -def steam_api() -> SteamAPI: - return SteamAPI() +# @pytest.fixture +# def steam_api() -> SteamAPI: +# return SteamAPI() -def test_get_feature_games(steam_api: SteamAPI): - assert steam_api.get_feature_games() +# def test_get_feature_games(steam_api: SteamAPI): +# assert steam_api.get_feature_games() -def test_get_game_screenshots(steam_api: SteamAPI, app_id: int): - assert steam_api.get_game_screenshots(app_id) +# def test_get_game_screenshots(steam_api: SteamAPI, app_id: int): +# assert steam_api.get_game_screenshots(app_id) -def test_get_game_details(steam_api: SteamAPI, app_id: int): - assert steam_api.get_game_details(app_id) +# def test_get_game_details(steam_api: SteamAPI, app_id: int): +# assert steam_api.get_game_details(app_id) -def test_get_game_details_kor(steam_api: SteamAPI, app_id: int): - assert steam_api.get_game_details(app_id, "korean") +# def test_get_game_details_kor(steam_api: SteamAPI, app_id: int): +# assert steam_api.get_game_details(app_id, "korean") -def test_get_game_details_from_gamalytic(steam_api: SteamAPI, app_id: int): - assert steam_api.get_game_details_from_gamalytic(app_id) +# def test_get_game_details_from_gamalytic(steam_api: SteamAPI, app_id: int): +# assert steam_api.get_game_details_from_gamalytic(app_id) -def test_get_game_details_no_content(steam_api: SteamAPI, no_detail_app_id: int): - with pytest.raises(SteamAPINoContentsException): - steam_api.get_game_details(no_detail_app_id) +# def test_get_game_details_no_content(steam_api: SteamAPI, no_detail_app_id: int): +# with pytest.raises(SteamAPINoContentsException): +# steam_api.get_game_details(no_detail_app_id)