diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml deleted file mode 100644 index 12aa9a30..00000000 --- a/.github/workflows/pylint.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Pylint - -on: - push: - branches: [master] - paths: - - "nazurin/**" - pull_request: - types: [opened, reopened, synchronize, edited] - branches: [master] - paths: - - "nazurin/**" - - ".github/workflows/pylint.yaml" - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - echo "::group::Install pip" - python -m pip install --upgrade pip - echo "::endgroup::" - echo "::group::Install dependencies" - pip install -r requirements.txt - echo "::endgroup::" - - name: Lint - uses: y-young/python-lint-annotate@v1 - with: - python-root-list: ./nazurin - extra-pylint-options: --fail-on=F,E --fail-under=9.5 diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 00000000..7d71465a --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,20 @@ +name: Ruff + +on: + push: + branches: [master] + paths: + - "nazurin/**" + pull_request: + types: [opened, reopened, synchronize, edited] + branches: [master] + paths: + - "nazurin/**" + - ".github/workflows/ruff.yaml" + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: y-young/ruff-action@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c021459e..b34e083f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,13 +9,12 @@ repos: - id: mixed-line-ending - id: trailing-whitespace files: \.py$ - - repo: https://github.com/pycqa/isort - rev: 5.13.2 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.5 hooks: - - id: isort - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 - hooks: - - id: black + - id: ruff + args: [--fix] + - id: ruff-format + ci: autoupdate_schedule: quarterly diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 1e49cd98..00000000 --- a/.pylintrc +++ /dev/null @@ -1,524 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Specify a score threshold to be exceeded before program exits with error. -fail-under=10.0 - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - missing-class-docstring, - missing-module-docstring, - missing-function-docstring, - protected-access, - too-few-public-methods - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Bad variable names regexes, separated by a comma. If names match any regex, -# they will always be refused -bad-names-rgxs= - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Good variable names regexes, separated by a comma. If names match any regex, -# they will always be accepted -# Allow short names and fix the bug that pylint cannot recognize -# constants inside function. -good-names-rgxs=^(?:[a-z][a-z0-9]*((_[a-z0-9]+)*)?|[A-Z][A-Z0-9]*((_[A-Z0-9]+)*)?)$ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# |'|\")??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=88 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - -# Regular expression of note tags to take in consideration. -#notes-rgx= - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no - -# This flag controls whether the implicit-str-concat should generate a warning -# on implicit string concatenation in sequences defined over several lines. -check-str-concat-over-line-jumps=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=builtins.BaseException, - builtins.Exception diff --git a/nazurin/__main__.py b/nazurin/__main__.py index c138f62e..3f714a9d 100644 --- a/nazurin/__main__.py +++ b/nazurin/__main__.py @@ -12,7 +12,7 @@ from nazurin import config, dp from nazurin.utils import logger from nazurin.utils.decorators import Cache, chat_action -from nazurin.utils.exceptions import InvalidCommandUsage, NazurinError +from nazurin.utils.exceptions import InvalidCommandUsageError, NazurinError from nazurin.utils.helpers import format_error @@ -39,14 +39,14 @@ async def show_help(message: Message, command: Command.CommandObj): 小さな小さな賢将, can help you collect images from various sites. Commands: - """ + """, ) + dp.commands.help_text() + dedent( """ PS: Send a URL of supported sites to collect image(s) - """ + """, ), ) @@ -89,7 +89,7 @@ async def on_error(update: Update, exception: Exception): message = update.message try: raise exception - except InvalidCommandUsage as error: + except InvalidCommandUsageError as error: await message.reply(dp.commands.help(error.command)) except ClientResponseError as error: traceback.print_exc() diff --git a/nazurin/bot.py b/nazurin/bot.py index 239a168b..642df2f0 100644 --- a/nazurin/bot.py +++ b/nazurin/bot.py @@ -26,7 +26,7 @@ class NazurinBot(Bot): send_message = retry_after(Bot.send_message) def __init__(self, *args, **kwargs): - super().__init__(parse_mode=ParseMode.HTML, *args, **kwargs) + super().__init__(*args, parse_mode=ParseMode.HTML, **kwargs) self.sites = SiteManager() self.storage = Storage() self.cleanup_task = None @@ -51,14 +51,16 @@ async def send_single_group( reply_to: Optional[int] = None, ): await self.send_chat_action(chat_id, ChatActions.UPLOAD_PHOTO) - media = [] - for img in imgs: - media.append(InputMediaPhoto(await img.display_url())) # TODO + # TODO: Fetch display URL in batch + media = [InputMediaPhoto(await img.display_url()) for img in imgs] media[0].caption = caption await self.send_media_group(chat_id, media, reply_to_message_id=reply_to) async def send_photos( - self, illust: Illust, chat_id: int, reply_to: Optional[int] = None + self, + illust: Illust, + chat_id: int, + reply_to: Optional[int] = None, ): caption = sanitize_caption(illust.caption) groups = [] @@ -101,11 +103,16 @@ async def send_illust( async def send_doc(self, file: File, chat_id, message_id=None): await self.send_chat_action(chat_id, ChatActions.UPLOAD_DOCUMENT) await self.send_document( - chat_id, InputFile(file.path), reply_to_message_id=message_id + chat_id, + InputFile(file.path), + reply_to_message_id=message_id, ) async def send_docs( - self, illust: Illust, message: Optional[Message] = None, chat_id=None + self, + illust: Illust, + message: Optional[Message] = None, + chat_id=None, ): if message: message_id = message.message_id @@ -117,7 +124,10 @@ async def send_docs( await self.send_doc(file, chat_id, message_id) async def send_to_gallery( - self, urls: List[str], illust: Illust, message: Optional[Message] = None + self, + urls: List[str], + illust: Illust, + message: Optional[Message] = None, ): if isinstance(illust, Ugoira): await self.send_illust(illust, message, config.GALLERY_ID) @@ -137,7 +147,9 @@ async def send_to_gallery( await self.send_illust(illust, message, config.GALLERY_ID) async def update_collection( - self, urls: List[str], message: Optional[Message] = None + self, + urls: List[str], + message: Optional[Message] = None, ): result = self.sites.match(urls) if not result: diff --git a/nazurin/commands.py b/nazurin/commands.py index eb8b5b7b..8ab425f3 100644 --- a/nazurin/commands.py +++ b/nazurin/commands.py @@ -26,13 +26,13 @@ def help(self) -> str: f""" {self.description} Usage: {self.usage} - """ + """, ) if self.help_text: text += dedent( f""" {self.help_text} - """ + """, ) return text @@ -77,7 +77,7 @@ def help_text(self) -> str: [ f"{command.usage} — {command.description}" for command in sorted(self.commands, key=lambda x: x.names[0]) - ] + ], ) def help(self, command: str) -> Optional[str]: diff --git a/nazurin/database/cloudant.py b/nazurin/database/cloudant.py index d7b97d0f..0378f799 100644 --- a/nazurin/database/cloudant.py +++ b/nazurin/database/cloudant.py @@ -1,4 +1,4 @@ -from cloudant.client import Cloudant as cloudant +from cloudant.client import Cloudant as CloudantBase from requests.adapters import HTTPAdapter from nazurin.config import RETRIES, env @@ -16,8 +16,11 @@ class Cloudant(DatabaseDriver): def __init__(self): """Connect to database.""" - self.client = cloudant.iam( - USERNAME, APIKEY, timeout=5, adapter=HTTPAdapter(max_retries=RETRIES) + self.client = CloudantBase.iam( + USERNAME, + APIKEY, + timeout=5, + adapter=HTTPAdapter(max_retries=RETRIES), ) self.client.connect() self.db = self.client[DATABASE] diff --git a/nazurin/database/firebase.py b/nazurin/database/firebase.py index de8d7c3a..0f5f40ee 100644 --- a/nazurin/database/firebase.py +++ b/nazurin/database/firebase.py @@ -1,4 +1,5 @@ import json +from typing import Optional import firebase_admin from firebase_admin import credentials, firestore_async @@ -33,7 +34,7 @@ def document(self, key=None): self._document = self._collection.document(str(key)) return self - async def list(self, page_size: int = None): + async def list(self, page_size: Optional[int] = None): return self._collection.list_documents(page_size) async def get(self): diff --git a/nazurin/database/mongo.py b/nazurin/database/mongo.py index b3328017..38f73624 100644 --- a/nazurin/database/mongo.py +++ b/nazurin/database/mongo.py @@ -13,8 +13,8 @@ class Mongo(DatabaseDriver): def __init__(self): """Load credentials and initialize client.""" - URI = env.str("MONGO_URI", default="mongodb://localhost:27017/nazurin") - self.client = AsyncIOMotorClient(URI) + uri = env.str("MONGO_URI", default="mongodb://localhost:27017/nazurin") + self.client = AsyncIOMotorClient(uri) self.db = self.client.get_default_database() self._collection = None self._document = None @@ -45,7 +45,8 @@ async def insert(self, key: Optional[Union[str, int]], data: dict) -> bool: async def update(self, data: dict) -> bool: result = await self._collection.update_one( - {"_id": self._document}, {"$set": data} + {"_id": self._document}, + {"$set": data}, ) return result.modified_count == 1 diff --git a/nazurin/dispatcher.py b/nazurin/dispatcher.py index 90aaa31a..ed4761f4 100644 --- a/nazurin/dispatcher.py +++ b/nazurin/dispatcher.py @@ -41,7 +41,7 @@ def init(self): content_types=[ContentType.TEXT, ContentType.PHOTO], ) - def message_handler( + def message_handler( # noqa: PLR0913 self, *custom_filters, commands=None, @@ -100,7 +100,8 @@ def start(self): if config.ENV == "production": logger.info("Set webhook") self.executor.set_webhook( - webhook_path="/" + config.TOKEN, web_app=self.server + webhook_path="/" + config.TOKEN, + web_app=self.server, ) # Tell aiohttp to use main thread event loop instead of creating a new one # otherwise bot commands will run in a different loop diff --git a/nazurin/middleware.py b/nazurin/middleware.py index f2fb2644..0be47c19 100644 --- a/nazurin/middleware.py +++ b/nazurin/middleware.py @@ -13,15 +13,17 @@ async def on_pre_process_message(self, message: Message, _data: dict): allowed_chats = config.ALLOW_ID + config.ALLOW_GROUP + [config.ADMIN_ID] if ( message.chat.id in allowed_chats - or message.from_user.id in config.ALLOW_ID + [config.ADMIN_ID] + or message.from_user.id in [*config.ALLOW_ID, config.ADMIN_ID] or message.from_user.username in config.ALLOW_USERNAME ): return - raise CancelHandler() + raise CancelHandler class LoggingMiddleware(BaseMiddleware): async def on_process_message(self, message: Message, _data: dict): logger.info( - "Message {}: {}", message.message_id, message.text or message.caption + "Message {}: {}", + message.message_id, + message.text or message.caption, ) diff --git a/nazurin/models/caption.py b/nazurin/models/caption.py index 45318ba7..3dc952fb 100644 --- a/nazurin/models/caption.py +++ b/nazurin/models/caption.py @@ -4,7 +4,7 @@ class Caption(dict): @property def text(self) -> str: - caption = str() + caption = "" for key, value in self.items(): if not value or key in CAPTION_IGNORE: continue diff --git a/nazurin/models/illust.py b/nazurin/models/illust.py index d51cfdfb..15bccbf7 100644 --- a/nazurin/models/illust.py +++ b/nazurin/models/illust.py @@ -30,7 +30,10 @@ def has_multiple_images(self) -> bool: return len(self.images) > 1 async def download( - self, *, request_class: NazurinRequestSession = Request, **kwargs + self, + *, + request_class: NazurinRequestSession = Request, + **kwargs, ): async with request_class(**kwargs) as session: files = filter(lambda file: file.url, self.all_files) diff --git a/nazurin/models/image.py b/nazurin/models/image.py index 85c84ef3..c6828487 100644 --- a/nazurin/models/image.py +++ b/nazurin/models/image.py @@ -11,6 +11,11 @@ from .file import File +TG_IMG_WIDTH_HEIGHT_RATIO_LIMIT = 20 +TG_IMG_DIMENSION_LIMIT = 10000 + +INVALID_IMAGE_RETRIES = 3 + @dataclass class Image(File): @@ -31,9 +36,13 @@ async def chosen_url(self) -> str: # https://core.telegram.org/bots/api#sendphoto if self._chosen_url: return self._chosen_url - if self.height != 0 and self.width / self.height > 20: + + if ( + self.height != 0 + and self.width / self.height > TG_IMG_WIDTH_HEIGHT_RATIO_LIMIT + ): raise NazurinError( - "Width and height ratio of image exceeds 20, try download option." + "Width and height ratio of image exceeds 20, try download option.", ) self._chosen_url = self.url if self.thumbnail: @@ -41,7 +50,7 @@ async def chosen_url(self) -> str: if ( (not self.width) or (not self.height) - or self.width + self.height > 10000 + or self.width + self.height > TG_IMG_DIMENSION_LIMIT ): self._chosen_url = self.thumbnail logger.info( @@ -66,15 +75,17 @@ async def size(self, **kwargs) -> int: self._size = self._size or await super().size() if self._size: return self._size - async with Request(**kwargs) as request: - async with request.head(self.url) as response: - headers = response.headers - if "Content-Length" in headers: - self._size = int(headers["Content-Length"]) - logger.info("Got image size: {}", naturalsize(self._size, True)) - else: - logger.info("Failed to get image size") - return self._size + async with Request(**kwargs) as request, request.head(self.url) as response: + headers = response.headers + if "Content-Length" in headers: + self._size = int(headers["Content-Length"]) + logger.info( + "Got image size: {}", + naturalsize(self._size, binary=True), + ) + else: + logger.info("Failed to get image size") + return self._size def __post_init__(self): if self._size: @@ -87,11 +98,10 @@ def set_size(self, value: int): self._size = int(value) async def download(self, session: aiohttp.ClientSession): - RETRIES = 3 - for i in range(RETRIES): + for i in range(INVALID_IMAGE_RETRIES): downloaded_size = await super().download(session) is_valid = await check_image(self.path) - attempt_count = f"{i + 1} / {RETRIES}" + attempt_count = f"{i + 1} / {INVALID_IMAGE_RETRIES}" if is_valid: if self._size is None or self._size == downloaded_size: return @@ -107,9 +117,9 @@ async def download(self, session: aiohttp.ClientSession): self.path, attempt_count, ) - if i < RETRIES - 1: + if i < INVALID_IMAGE_RETRIES - 1: # Keep the last one for debugging os.remove(self.path) raise NazurinError( - "Download failed with invalid image, please check logs for details" + "Download failed with invalid image, please check logs for details", ) diff --git a/nazurin/models/ugoira.py b/nazurin/models/ugoira.py index ebc0217c..30131b85 100644 --- a/nazurin/models/ugoira.py +++ b/nazurin/models/ugoira.py @@ -1,6 +1,5 @@ -import os from dataclasses import dataclass -from typing import Optional, Union +from typing import Union from .caption import Caption from .file import File @@ -11,8 +10,13 @@ class Ugoira(Illust): video: File = None - def __init__( - self, id: Union[int, str], video, caption=None, metadata=None, files=None + def __init__( # noqa: PLR0913 + self, + id: Union[int, str], + video, + caption=None, + metadata=None, + files=None, ): super().__init__(id) self.video = video @@ -22,7 +26,7 @@ def __init__( @property def all_files(self): - return [self.video] + self.files + return [self.video, *self.files] def has_image(self) -> bool: return self.video is not None diff --git a/nazurin/server.py b/nazurin/server.py index 07559063..400b4808 100644 --- a/nazurin/server.py +++ b/nazurin/server.py @@ -23,8 +23,9 @@ def __init__(self, bot): resource.add_route("POST", self.update_handler), { "*": aiohttp_cors.ResourceOptions( - allow_headers=("Content-Type",), allow_methods=["POST", "OPTIONS"] - ) + allow_headers=("Content-Type",), + allow_methods=["POST", "OPTIONS"], + ), }, ) setup_jobs(self) @@ -38,11 +39,13 @@ async def do_update(self, url): logger.info("API request: {}", url) await self.bot.update_collection([url]) await self.bot.send_message( - config.ADMIN_ID, f"Successfully collected {url}" + config.ADMIN_ID, + f"Successfully collected {url}", ) except NazurinError as error: await self.bot.send_message( - config.ADMIN_ID, f"Error processing {url}: {error}" + config.ADMIN_ID, + f"Error processing {url}: {error}", ) # pylint: disable-next=broad-exception-caught except Exception as error: @@ -50,7 +53,8 @@ async def do_update(self, url): if isinstance(error, asyncio.TimeoutError): error = "Timeout, please try again." await self.bot.send_message( - config.ADMIN_ID, f"Error processing {url}: {format_error(error)}" + config.ADMIN_ID, + f"Error processing {url}: {format_error(error)}", ) async def update_handler(self, request): diff --git a/nazurin/sites/__init__.py b/nazurin/sites/__init__.py index d2e605a2..e0460307 100644 --- a/nazurin/sites/__init__.py +++ b/nazurin/sites/__init__.py @@ -49,19 +49,20 @@ def load(self): module = import_module("nazurin.sites." + module_name) # Store site API class self.sites[module_name.lower()] = getattr( - module, snake_to_pascal(module_name) + module, + snake_to_pascal(module_name), )() if hasattr(module, "patterns") and hasattr(module, "handle"): - PRIORITY = getattr(module, "PRIORITY") - patterns = getattr(module, "patterns") - handle = getattr(module, "handle") + priority = module.PRIORITY + patterns = module.patterns + handle = module.handle self.sources.append( Source( - priority=PRIORITY, + priority=priority, patterns=patterns, handler=handle, name=module_name, - ) + ), ) self.sources.sort(key=lambda s: s.priority, reverse=True) logger.info("Loaded {} sites", len(self.sites)) diff --git a/nazurin/sites/artstation/api.py b/nazurin/sites/artstation/api.py index c633718b..105d6630 100644 --- a/nazurin/sites/artstation/api.py +++ b/nazurin/sites/artstation/api.py @@ -1,5 +1,6 @@ import os from datetime import datetime +from http import HTTPStatus from typing import List, Tuple from nazurin.models import Caption, Illust, Image @@ -15,13 +16,12 @@ class Artstation: async def get_post(self, post_id: str): """Fetch a post.""" api = f"https://www.artstation.com/projects/{post_id}.json" - async with Request() as request: - async with request.get(api) as response: - if response.status == 404: - raise NazurinError("Post not found") - response.raise_for_status() - post = await response.json() - return post + async with Request() as request, request.get(api) as response: + if response.status == HTTPStatus.NOT_FOUND: + raise NazurinError("Post not found") + response.raise_for_status() + post = await response.json() + return post async def fetch(self, post_id: str) -> Illust: post = await self.get_post(post_id) @@ -50,7 +50,7 @@ def get_images(self, post) -> List[Image]: thumbnail, width=asset["width"], height=asset["height"], - ) + ), ) index += 1 return imgs @@ -79,7 +79,7 @@ def get_storage_dest(post: dict, asset: dict, index: int = 0) -> Tuple[str, str] def build_caption(post) -> Caption: user = post["user"] tags = post["tags"] - tag_string = str() + tag_string = "" for tag in tags: tag_string += "#" + tag + " " return Caption( @@ -88,7 +88,7 @@ def build_caption(post) -> Caption: "author": f"{user['full_name']} #{user['username']}", "url": f"https://www.artstation.com/artwork/{post['hash_id']}", "tags": tag_string, - } + }, ) @staticmethod diff --git a/nazurin/sites/artstation/config.py b/nazurin/sites/artstation/config.py index aa187587..c50d50ee 100644 --- a/nazurin/sites/artstation/config.py +++ b/nazurin/sites/artstation/config.py @@ -3,7 +3,6 @@ PRIORITY = 10 COLLECTION = "artstation" -with env.prefixed("ARTSTATION_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Artstation") - FILENAME: str = env.str("NAME", default="{title} ({hash_id}) - {filename}") +with env.prefixed("ARTSTATION_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Artstation") + FILENAME: str = env.str("NAME", default="{title} ({hash_id}) - {filename}") diff --git a/nazurin/sites/artstation/interface.py b/nazurin/sites/artstation/interface.py index f42b32bf..0dd6026f 100644 --- a/nazurin/sites/artstation/interface.py +++ b/nazurin/sites/artstation/interface.py @@ -9,7 +9,7 @@ patterns = [ # https://www.artstation.com/artwork/2x3LaB # https://catzz.artstation.com/projects/A9ELeq - r"artstation\.com/(?:artwork|projects)/([0-9a-zA-Z]+)" + r"artstation\.com/(?:artwork|projects)/([0-9a-zA-Z]+)", ] diff --git a/nazurin/sites/bilibili/api.py b/nazurin/sites/bilibili/api.py index a549616e..ff88850e 100644 --- a/nazurin/sites/bilibili/api.py +++ b/nazurin/sites/bilibili/api.py @@ -1,5 +1,5 @@ import os -from datetime import datetime +from datetime import datetime, timezone from typing import List, Tuple from nazurin.models import Caption, Illust, Image @@ -9,6 +9,8 @@ from .config import DESTINATION, FILENAME +ERROR_NOT_FOUND = 4101147 + class Bilibili: @network_retry @@ -17,19 +19,18 @@ async def get_dynamic(self, dynamic_id: str): api = ( f"https://api.bilibili.com/x/polymer/web-dynamic/v1/detail?id={dynamic_id}" ) - async with Request() as request: - async with request.get(api) as response: - response.raise_for_status() - data = await response.json() - # For some IDs, the API returns code 0 but empty content - code = data.get("code") - if code == 4101147 or "data" not in data: - raise NazurinError("Dynamic not found") - if code != 0: - raise NazurinError( - f"Failed to get dynamic: code = {code}, " - f"message = {data['message']}" - ) + async with Request() as request, request.get(api) as response: + response.raise_for_status() + data = await response.json() + # For some IDs, the API returns code 0 but empty content + code = data.get("code") + if code == ERROR_NOT_FOUND or "data" not in data: + raise NazurinError("Dynamic not found") + if code != 0: + raise NazurinError( + f"Failed to get dynamic: code = {code}, " + f"message = {data['message']}", + ) item = data["data"]["item"] return self.cleanup_item(item) @@ -68,7 +69,7 @@ def get_images(item: dict) -> List[Image]: size, pic["width"], pic["height"], - ) + ), ) return imgs @@ -79,7 +80,10 @@ def get_storage_dest(item: dict, pic: dict, index: int = 0) -> Tuple[str, str]: """ url = pic["src"] - timestamp = datetime.fromtimestamp(item["modules"]["module_author"]["pub_ts"]) + timestamp = datetime.fromtimestamp( + item["modules"]["module_author"]["pub_ts"], + tz=timezone.utc, + ) basename = os.path.basename(url) filename, extension = os.path.splitext(basename) user = item["modules"]["module_author"] @@ -106,7 +110,7 @@ def build_caption(item: dict) -> Caption: { "author": "#" + modules["module_author"]["name"], "content": modules["module_dynamic"]["desc"]["text"], - } + }, ) @staticmethod diff --git a/nazurin/sites/bilibili/config.py b/nazurin/sites/bilibili/config.py index c70b4f69..331f9e88 100644 --- a/nazurin/sites/bilibili/config.py +++ b/nazurin/sites/bilibili/config.py @@ -3,9 +3,9 @@ PRIORITY = 4 COLLECTION = "bilibili" -with env.prefixed("BILIBILI_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Bilibili") - FILENAME: str = env.str( - "NAME", default="{id_str}_{index} - {user[name]}({user[mid]})" - ) +with env.prefixed("BILIBILI_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Bilibili") + FILENAME: str = env.str( + "NAME", + default="{id_str}_{index} - {user[name]}({user[mid]})", + ) diff --git a/nazurin/sites/bluesky/api.py b/nazurin/sites/bluesky/api.py index f5c4219a..7b15fd1e 100644 --- a/nazurin/sites/bluesky/api.py +++ b/nazurin/sites/bluesky/api.py @@ -18,13 +18,15 @@ async def resolve_handle(self, handle: str): https://www.docs.bsky.app/docs/api/com-atproto-identity-resolve-handle """ api = "https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle" - async with Request() as request: - async with request.get(api, params={"handle": handle}) as response: - data = await response.json() - if "error" in data: - raise NazurinError(data["message"]) - response.raise_for_status() - return data["did"] + async with Request() as request, request.get( + api, + params={"handle": handle}, + ) as response: + data = await response.json() + if "error" in data: + raise NazurinError(data["message"]) + response.raise_for_status() + return data["did"] @network_retry async def get_post_thread(self, uri: str, depth: int, parent_height: int): @@ -34,13 +36,12 @@ async def get_post_thread(self, uri: str, depth: int, parent_height: int): """ api = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread" params = {"uri": uri, "depth": depth, "parentHeight": parent_height} - async with Request() as request: - async with request.get(api, params=params) as response: - data = await response.json() - if "error" in data: - raise NazurinError(data["message"]) - response.raise_for_status() - return data["thread"]["post"] + async with Request() as request, request.get(api, params=params) as response: + data = await response.json() + if "error" in data: + raise NazurinError(data["message"]) + response.raise_for_status() + return data["thread"]["post"] async def fetch(self, user_handle: str, post_rkey: str) -> Illust: """Fetch images and detail.""" @@ -78,7 +79,7 @@ def get_images(item: dict) -> List[Image]: url, destination, thumbnail, - ) + ), ) return imgs @@ -123,5 +124,5 @@ def build_caption(item: dict) -> Caption: { "author": "#" + item["author"]["displayName"], "text": item["record"]["text"], - } + }, ) diff --git a/nazurin/sites/bluesky/config.py b/nazurin/sites/bluesky/config.py index 36cb6362..d698969b 100644 --- a/nazurin/sites/bluesky/config.py +++ b/nazurin/sites/bluesky/config.py @@ -3,9 +3,9 @@ PRIORITY = 10 COLLECTION = "bluesky" -with env.prefixed("BLUESKY_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Bluesky") - FILENAME: str = env.str( - "NAME", default="{rkey}_{index} - {user[display_name]}({user[handle]})" - ) +with env.prefixed("BLUESKY_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Bluesky") + FILENAME: str = env.str( + "NAME", + default="{rkey}_{index} - {user[display_name]}({user[handle]})", + ) diff --git a/nazurin/sites/bluesky/interface.py b/nazurin/sites/bluesky/interface.py index 5d7c1083..1a49810c 100644 --- a/nazurin/sites/bluesky/interface.py +++ b/nazurin/sites/bluesky/interface.py @@ -9,7 +9,7 @@ patterns = [ # https://atproto.com/specs/record-key#record-key-syntax # https://bsky.app/profile/shiratamacaron.bsky.social/post/3kkt7oj5rmw2j - r"bsky\.app/profile/([\w\.\-]+)/post/([\w\.\-~]+)" + r"bsky\.app/profile/([\w\.\-]+)/post/([\w\.\-~]+)", ] diff --git a/nazurin/sites/danbooru/__init__.py b/nazurin/sites/danbooru/__init__.py index c553e506..4aa83016 100644 --- a/nazurin/sites/danbooru/__init__.py +++ b/nazurin/sites/danbooru/__init__.py @@ -1,7 +1,7 @@ """Danbooru site plugin.""" from .api import Danbooru -from .commands import * +from .commands import * # noqa: F403 from .config import PRIORITY from .interface import handle, patterns diff --git a/nazurin/sites/danbooru/api.py b/nazurin/sites/danbooru/api.py index 4610cb80..cb85da39 100644 --- a/nazurin/sites/danbooru/api.py +++ b/nazurin/sites/danbooru/api.py @@ -4,7 +4,7 @@ from os import path from typing import List, Optional, Tuple -from pybooru import Danbooru as danbooru +from pybooru import Danbooru as DanbooruBase from pybooru import PybooruHTTPError from nazurin.models import Caption, File, Image @@ -15,17 +15,21 @@ from .config import DESTINATION, FILENAME +MAX_CHARACTER_COUNT = 5 + class Danbooru: def __init__(self, site="danbooru"): """Set Danbooru site.""" self.site = site - self.api = danbooru(site) + self.api = DanbooruBase(site) self.post_show = async_wrap(self.api.post_show) self.post_list = async_wrap(self.api.post_list) async def get_post( - self, post_id: Optional[int] = None, md5: Optional[str] = None + self, + post_id: Optional[int] = None, + md5: Optional[str] = None, ) -> dict: """Fetch a post.""" try: @@ -40,12 +44,14 @@ async def get_post( if "file_url" not in post: raise NazurinError( "You may need a gold account to view this post\nSource: " - + post["source"] + + post["source"], ) return post async def view( - self, post_id: Optional[int] = None, md5: Optional[str] = None + self, + post_id: Optional[int] = None, + md5: Optional[str] = None, ) -> DanbooruIllust: post = await self.get_post(post_id, md5) illust = self.parse_post(post) @@ -70,14 +76,14 @@ def parse_post(self, post: dict) -> DanbooruIllust: post["file_size"], post["image_width"], post["image_height"], - ) + ), ) else: # danbooru has non-image posts, such as #animated files.append(File(filename, url, destination)) # Build media caption tags = post["tag_string"].split(" ") - tag_string = str() + tag_string = "" for character in tags: tag_string += "#" + character + " " caption = Caption( @@ -89,7 +95,7 @@ def parse_post(self, post: dict) -> DanbooruIllust: "parent_id": post["parent_id"], "pixiv_id": post["pixiv_id"], "has_children": post["has_children"], - } + }, ) return DanbooruIllust(int(post["id"]), imgs, caption, post, files) @@ -126,7 +132,7 @@ def _get_names(post) -> Tuple[str, str]: copyrights = Danbooru._format_copyrights(post["tag_string_copyright"]) artists = Danbooru._format_artists(post["tag_string_artist"]) extension = path.splitext(post["file_url"])[1] - filename = str() + filename = "" if characters: filename += characters + " " @@ -146,10 +152,10 @@ def _format_characters(characters: str) -> str: characters = characters.split(" ") characters = list(map(Danbooru._normalize, characters)) size = len(characters) - if size <= 5: + if size <= MAX_CHARACTER_COUNT: result = Danbooru._sentence(characters) else: - characters = characters[:5] + characters = characters[:MAX_CHARACTER_COUNT] result = Danbooru._sentence(characters) + " and " + str(size - 1) + " more" return result diff --git a/nazurin/sites/danbooru/commands.py b/nazurin/sites/danbooru/commands.py index 744232fc..9e7ab353 100644 --- a/nazurin/sites/danbooru/commands.py +++ b/nazurin/sites/danbooru/commands.py @@ -2,7 +2,7 @@ from aiogram.types import Message from nazurin import bot, dp -from nazurin.utils.exceptions import InvalidCommandUsage +from nazurin.utils.exceptions import InvalidCommandUsageError from .api import Danbooru @@ -10,13 +10,15 @@ @dp.message_handler( - Command(["danbooru"]), args="POST_ID", description="View Danbooru post" + Command(["danbooru"]), + args="POST_ID", + description="View Danbooru post", ) async def danbooru_view(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("danbooru") from e + raise InvalidCommandUsageError("danbooru") from e if post_id <= 0: await message.reply("Invalid post id!") return @@ -25,13 +27,15 @@ async def danbooru_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["danbooru_download"]), args="POST_ID", description="Download Danbooru post" + Command(["danbooru_download"]), + args="POST_ID", + description="Download Danbooru post", ) async def danbooru_download(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("danbooru_download") from e + raise InvalidCommandUsageError("danbooru_download") from e if post_id <= 0: await message.reply("Invalid post id!") return diff --git a/nazurin/sites/danbooru/config.py b/nazurin/sites/danbooru/config.py index 1cce8184..613070a0 100644 --- a/nazurin/sites/danbooru/config.py +++ b/nazurin/sites/danbooru/config.py @@ -3,7 +3,6 @@ PRIORITY = 30 COLLECTION = "danbooru" -with env.prefixed("DANBOORU_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Danbooru") - FILENAME: str = env.str("NAME", default="{id} - {filename}") +with env.prefixed("DANBOORU_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Danbooru") + FILENAME: str = env.str("NAME", default="{id} - {filename}") diff --git a/nazurin/sites/deviant_art/api.py b/nazurin/sites/deviant_art/api.py index d363651c..bc0cb233 100644 --- a/nazurin/sites/deviant_art/api.py +++ b/nazurin/sites/deviant_art/api.py @@ -3,6 +3,7 @@ import os import re from datetime import datetime +from http import HTTPStatus from http.cookies import SimpleCookie from typing import List, Optional, Tuple from urllib.parse import urlparse @@ -23,9 +24,9 @@ class DeviantArt: cookies: SimpleCookie = None @network_retry - async def get_deviation(self, deviation_id: int, retry: bool = False) -> dict: + async def get_deviation(self, deviation_id: int, *, retry: bool = False) -> dict: """Fetch a deviation.""" - await self.require_csrf_token(retry) + await self.require_csrf_token(refresh=retry) api = f"{BASE_URL}/_napi/da-user-profile/shared_api/deviation/extended_fetch" params = { "type": "art", @@ -33,28 +34,25 @@ async def get_deviation(self, deviation_id: int, retry: bool = False) -> dict: "csrf_token": self.csrf_token, } async with Request( - cookies=DeviantArt.cookies, headers={"Referer": BASE_URL} - ) as request: - async with request.get(api, params=params) as response: - if response.status == 404: - raise NazurinError("Deviation not found") - response.raise_for_status() - - data = await response.json() - if "error" in data: - logger.error(data) - # If CSRF token is invalid, try to get a new one - if ( - data.get("errorDetails", {}).get("csrf") == "invalid" - and not retry - ): - logger.info("CSRF token seems expired, refreshing...") - return await self.get_deviation(deviation_id, True) - raise NazurinError(data["errorDescription"]) - - deviation = data["deviation"] - del deviation["extended"]["relatedContent"] - return deviation + cookies=DeviantArt.cookies, + headers={"Referer": BASE_URL}, + ) as request, request.get(api, params=params) as response: + if response.status == HTTPStatus.NOT_FOUND: + raise NazurinError("Deviation not found") + response.raise_for_status() + + data = await response.json() + if "error" in data: + logger.error(data) + # If CSRF token is invalid, try to get a new one + if data.get("errorDetails", {}).get("csrf") == "invalid" and not retry: + logger.info("CSRF token seems expired, refreshing...") + return await self.get_deviation(deviation_id, retry=True) + raise NazurinError(data["errorDescription"]) + + deviation = data["deviation"] + del deviation["extended"]["relatedContent"] + return deviation async def fetch(self, deviation_id: str) -> Illust: deviation = await self.get_deviation(deviation_id) @@ -77,13 +75,16 @@ def get_images(self, deviation: dict) -> List[Image]: original_file["filesize"], original_file["width"], original_file["height"], - ) + ), ] return imgs @staticmethod def get_storage_dest( - deviation: dict, filename: str, is_download: bool = False + deviation: dict, + filename: str, + *, + is_download: bool = False, ) -> Tuple[str, str]: """ Format destination and filename. @@ -136,7 +137,11 @@ async def get_download(self, deviation: dict) -> Optional[File]: # Duplicate attribute on top level for convenience deviation["prettyName"] = deviation["media"]["prettyName"] - destination, filename = self.get_storage_dest(deviation, filename, True) + destination, filename = self.get_storage_dest( + deviation, + filename, + is_download=True, + ) return File(filename, f"{url.geturl()}?token={token}", destination) @staticmethod @@ -146,11 +151,11 @@ def build_caption(deviation: dict) -> Caption: "title": deviation["title"], "author": f"#{deviation['author']['username']}", "url": deviation["url"], - } + }, ) if "tags" in deviation["extended"]: caption["tags"] = " ".join( - ["#" + tag["name"] for tag in deviation["extended"]["tags"]] + ["#" + tag["name"] for tag in deviation["extended"]["tags"]], ) return caption @@ -161,7 +166,7 @@ def parse_url(self, deviation: dict) -> Tuple[str, str, str]: media = deviation["media"] base_uri = media["baseUri"] - tokens = media["token"] if "token" in media else [] + tokens = media.get("token", []) types = {} for type_ in media["types"]: types[type_["t"]] = type_ @@ -183,7 +188,8 @@ def parse_url(self, deviation: dict) -> Tuple[str, str, str]: else: thumbnail = types["preview"] thumbnail = base_uri + thumbnail["c"].replace( - "", media["prettyName"] + "", + media["prettyName"], ) thumbnail = f"{thumbnail}?token={token}" elif base_uri.endswith(".gif"): # TODO: Send GIFs properly @@ -207,24 +213,23 @@ def generate_token(path: str) -> str: return f"{header}.{payload}." @network_retry - async def require_csrf_token(self, refresh: bool = False) -> None: + async def require_csrf_token(self, *, refresh: bool = False) -> None: if self.csrf_token and not refresh: return logger.info("Fetching CSRF token...") - async with Request() as request: - async with request.get(BASE_URL) as response: - response.raise_for_status() - pattern = re.compile(r"window\.__CSRF_TOKEN__ = '(\S+)';") - content = await response.text() - match = pattern.search(content) - if not match: - raise NazurinError("Unable to get CSRF token") - DeviantArt.csrf_token = match.group(1) - # CSRF token must be used along with the cookies returned, - # otherwise will be considered invalid - DeviantArt.cookies = response.cookies - logger.info( - "Fetched CSRF token: {}, cookies: {}", - DeviantArt.csrf_token, - DeviantArt.cookies, - ) + async with Request() as request, request.get(BASE_URL) as response: + response.raise_for_status() + pattern = re.compile(r"window\.__CSRF_TOKEN__ = '(\S+)';") + content = await response.text() + match = pattern.search(content) + if not match: + raise NazurinError("Unable to get CSRF token") + DeviantArt.csrf_token = match.group(1) + # CSRF token must be used along with the cookies returned, + # otherwise will be considered invalid + DeviantArt.cookies = response.cookies + logger.info( + "Fetched CSRF token: {}, cookies: {}", + DeviantArt.csrf_token, + DeviantArt.cookies, + ) diff --git a/nazurin/sites/deviant_art/config.py b/nazurin/sites/deviant_art/config.py index 6503c4dc..d28efee3 100644 --- a/nazurin/sites/deviant_art/config.py +++ b/nazurin/sites/deviant_art/config.py @@ -3,10 +3,10 @@ PRIORITY = 10 COLLECTION = "deviantart" -with env.prefixed("DEVIANT_ART_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="DeviantArt") - FILENAME: str = env.str("NAME", default="{title} - {deviationId}") - DOWNLOAD_FILENAME: str = env.str( - "DOWNLOAD_NAME", default="{title} - {deviationId} - {prettyName}" - ) +with env.prefixed("DEVIANT_ART_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="DeviantArt") + FILENAME: str = env.str("NAME", default="{title} - {deviationId}") + DOWNLOAD_FILENAME: str = env.str( + "DOWNLOAD_NAME", + default="{title} - {deviationId} - {prettyName}", + ) diff --git a/nazurin/sites/gelbooru/api.py b/nazurin/sites/gelbooru/api.py index 804000c6..037a2bd2 100644 --- a/nazurin/sites/gelbooru/api.py +++ b/nazurin/sites/gelbooru/api.py @@ -17,14 +17,13 @@ async def get_post(self, post_id: int): "https://gelbooru.com/index.php?page=dapi&s=post&q=index&json=1&id=" + str(post_id) ) - async with Request() as request: - async with request.get(api) as response: - response.raise_for_status() - response = await response.json() - if "post" not in response: - raise NazurinError("Post not found") - post = response["post"][0] - return post + async with Request() as request, request.get(api) as response: + response.raise_for_status() + response_json = await response.json() + if "post" not in response_json: + raise NazurinError("Post not found") + post = response_json["post"][0] + return post async def fetch(self, post_id: int) -> Illust: post = await self.get_post(post_id) @@ -45,7 +44,7 @@ def get_images(self, post) -> List[Image]: self.get_thumbnail(post), width=post["width"], height=post["height"], - ) + ), ) return imgs @@ -70,7 +69,7 @@ def get_storage_dest(post: dict) -> Tuple[str, str]: @staticmethod def build_caption(post) -> Caption: tags = post["tags"].split(" ") - tag_string = str() + tag_string = "" for tag in tags: tag_string += "#" + tag + " " return Caption( @@ -80,7 +79,7 @@ def build_caption(post) -> Caption: "url": "https://gelbooru.com/index.php" f"?page=post&s=view&id={post['id']}", "tags": tag_string, - } + }, ) @staticmethod diff --git a/nazurin/sites/gelbooru/config.py b/nazurin/sites/gelbooru/config.py index 421e919a..3395d4a3 100644 --- a/nazurin/sites/gelbooru/config.py +++ b/nazurin/sites/gelbooru/config.py @@ -3,7 +3,6 @@ PRIORITY = 8 COLLECTION = "gelbooru" -with env.prefixed("GELBOORU_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Gelbooru") - FILENAME: str = env.str("NAME", default="{id}") +with env.prefixed("GELBOORU_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Gelbooru") + FILENAME: str = env.str("NAME", default="{id}") diff --git a/nazurin/sites/gelbooru/interface.py b/nazurin/sites/gelbooru/interface.py index 42a50071..86ee74b6 100644 --- a/nazurin/sites/gelbooru/interface.py +++ b/nazurin/sites/gelbooru/interface.py @@ -8,7 +8,7 @@ patterns = [ # https://gelbooru.com/index.php?page=post&s=view&id=123456 - r"gelbooru\.com/index\.php\?page=post&s=view&id=(\d+)" + r"gelbooru\.com/index\.php\?page=post&s=view&id=(\d+)", ] diff --git a/nazurin/sites/kemono/api.py b/nazurin/sites/kemono/api.py index 9b824412..72f8aa0b 100644 --- a/nazurin/sites/kemono/api.py +++ b/nazurin/sites/kemono/api.py @@ -21,53 +21,58 @@ class Kemono: async def get_post(self, service: str, user_id: str, post_id: str) -> dict: """Fetch an post.""" api = f"{self.API_BASE}/{service}/user/{user_id}/post/{post_id}" - async with Request() as request: - async with request.get(api) as response: - response.raise_for_status() - post = await response.json() - if not post: - raise NazurinError("Post not found") - username = await self.get_username(service, user_id) - post["username"] = username - return post + async with Request() as request, request.get(api) as response: + response.raise_for_status() + post = await response.json() + if not post: + raise NazurinError("Post not found") + username = await self.get_username(service, user_id) + post["username"] = username + return post @network_retry async def get_post_revision( - self, service: str, user_id: str, post_id: str, revision_id: str + self, + service: str, + user_id: str, + post_id: str, + revision_id: str, ) -> dict: """Fetch a post revision.""" api = f"{self.API_BASE}/{service}/user/{user_id}/post/{post_id}/revisions" - async with Request() as request: - async with request.get(api) as response: - response.raise_for_status() - revisions = await response.json() - post = None - for revision in revisions: - if str(revision["revision_id"]) == revision_id: - post = revision - break - if not post: - raise NazurinError("Post revision not found") - username = await self.get_username(service, user_id) - post["username"] = username - return post + async with Request() as request, request.get(api) as response: + response.raise_for_status() + revisions = await response.json() + post = None + for revision in revisions: + if str(revision["revision_id"]) == revision_id: + post = revision + break + if not post: + raise NazurinError("Post revision not found") + username = await self.get_username(service, user_id) + post["username"] = username + return post @network_retry async def get_username(self, service: str, user_id: str) -> str: url = f"https://kemono.su/{service}/user/{user_id}" - async with Request() as request: - async with request.get(url) as response: - response.raise_for_status() - response = await response.text() - soup = BeautifulSoup(response, "html.parser") - tag = soup.find("meta", attrs={"name": "artist_name"}) - if not tag: - return "" - username = tag.get("content", "") - return username + async with Request() as request, request.get(url) as response: + response.raise_for_status() + response_text = await response.text() + soup = BeautifulSoup(response_text, "html.parser") + tag = soup.find("meta", attrs={"name": "artist_name"}) + if not tag: + return "" + username = tag.get("content", "") + return username async def fetch( - self, service: str, user_id: str, post_id: str, revision_id: Union[str, None] + self, + service: str, + user_id: str, + post_id: str, + revision_id: Union[str, None], ) -> Illust: if revision_id: post = await self.get_post_revision(service, user_id, post_id, revision_id) @@ -98,7 +103,9 @@ async def fetch( # Handle images destination, filename = self.get_storage_dest( - post, f"{image_index} - {file['name']}", path + post, + f"{image_index} - {file['name']}", + path, ) thumbnail = "https://img.kemono.su/thumbnail/data" + path images.append( @@ -107,7 +114,7 @@ async def fetch( url, destination, thumbnail, - ) + ), ) image_index += 1 @@ -160,7 +167,7 @@ def build_caption(post) -> Caption: "title": post["title"], "author": "#" + post["username"], "url": Kemono.get_url(post), - } + }, ) @staticmethod diff --git a/nazurin/sites/kemono/config.py b/nazurin/sites/kemono/config.py index aa5ae5a2..c443ef05 100644 --- a/nazurin/sites/kemono/config.py +++ b/nazurin/sites/kemono/config.py @@ -3,9 +3,9 @@ PRIORITY = 10 COLLECTION = "kemono" -with env.prefixed("KEMONO_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str( - "PATH", default="Kemono/{service}/{username} ({user})/{title} ({id})" - ) - FILENAME: str = env.str("NAME", default="{pretty_name}") +with env.prefixed("KEMONO_"), env.prefixed("FILE_"): + DESTINATION: str = env.str( + "PATH", + default="Kemono/{service}/{username} ({user})/{title} ({id})", + ) + FILENAME: str = env.str("NAME", default="{pretty_name}") diff --git a/nazurin/sites/kemono/interface.py b/nazurin/sites/kemono/interface.py index d195e0bb..2cdb3cb9 100644 --- a/nazurin/sites/kemono/interface.py +++ b/nazurin/sites/kemono/interface.py @@ -15,7 +15,7 @@ # https://kemono.party/gumroad/user/12345/post/aBc1d2 # https://kemono.su/subscribestar/user/abcdef/post/12345 # https://kemono.su/fanbox/user/12345/post/12345/revision/12345 - r"kemono\.(?:party|su)/(\w+)/user/([\w-]+)/post/([\w-]+)(?:/revision/(\d+))?" + r"kemono\.(?:party|su)/(\w+)/user/([\w-]+)/post/([\w-]+)(?:/revision/(\d+))?", ] diff --git a/nazurin/sites/lofter/api.py b/nazurin/sites/lofter/api.py index daffdec0..e05031a8 100644 --- a/nazurin/sites/lofter/api.py +++ b/nazurin/sites/lofter/api.py @@ -1,6 +1,7 @@ import json import os -from datetime import datetime +from datetime import datetime, timezone +from http import HTTPStatus from typing import List, Tuple from urllib.parse import parse_qs, urlparse @@ -15,28 +16,28 @@ class Lofter: + API = "https://api.lofter.com/oldapi/post/detail.api" + UA = "LOFTER/6.24.0 (iPhone; iOS 15.4.1; Scale/3.00)" + @network_retry async def get_post(self, username: str, permalink: str) -> dict: """Fetch a post.""" (blog_id, post_id) = await self.get_real_id(username, permalink) - API = "https://api.lofter.com/oldapi/post/detail.api" - UA = "LOFTER/6.24.0 (iPhone; iOS 15.4.1; Scale/3.00)" - async with Request(headers={"User-Agent": UA}) as request: - async with request.post( - API, - data={ - "targetblogid": blog_id, - "postid": post_id, - }, - ) as response: - response.raise_for_status() + async with Request(headers={"User-Agent": self.UA}) as request, request.post( + self.API, + data={ + "targetblogid": blog_id, + "postid": post_id, + }, + ) as response: + response.raise_for_status() - data = await response.json() - if data["meta"]["status"] != 200: - raise NazurinError(data["meta"]["msg"]) + data = await response.json() + if data["meta"]["status"] != HTTPStatus.OK: + raise NazurinError(data["meta"]["msg"]) - post = data["response"]["posts"][0]["post"] - return post + post = data["response"]["posts"][0]["post"] + return post async def fetch(self, username: str, permalink: str) -> Illust: post = await self.get_post(username, permalink) @@ -48,7 +49,7 @@ async def fetch(self, username: str, permalink: str) -> Illust: + post["blogInfo"]["blogName"], "url": post["blogPageUrl"], "tags": " ".join(["#" + tag for tag in post["tagList"]]), - } + }, ) return Illust(int(post["id"]), imgs, caption, post) @@ -72,7 +73,7 @@ def get_images(post: dict) -> List[Image]: thumbnail=photo["orign"], width=photo["rw"], height=photo["rh"], - ) + ), ) return imgs @@ -82,7 +83,10 @@ def get_storage_dest(post: dict, filename: str, index: int) -> Tuple[str, str]: Format destination and filename. """ - publish_time = datetime.fromtimestamp(post["publishTime"] / 1000) + publish_time = datetime.fromtimestamp( + post["publishTime"] / 1000, + tz=timezone.utc, + ) filename, extension = os.path.splitext(filename) context = { **post, @@ -101,17 +105,16 @@ def get_storage_dest(post: dict, filename: str, index: int) -> Tuple[str, str]: async def get_real_id(self, username: str, post_id: str) -> Tuple[int, int]: """Get real numeric blog ID and post ID.""" api = f"https://{username}.lofter.com/post/{post_id}" - async with Request() as request: - async with request.get(api) as response: - if response.status == 404: - raise NazurinError("Post not found") - response.raise_for_status() + async with Request() as request, request.get(api) as response: + if response.status == HTTPStatus.NOT_FOUND: + raise NazurinError("Post not found") + response.raise_for_status() - response = await response.text() - soup = BeautifulSoup(response, "html.parser") - iframe = soup.find("iframe", id="control_frame") - if not iframe: - raise NazurinError("Failed to get real post ID") - src = urlparse(iframe.get("src")) - query = parse_qs(src.query) - return (query["blogId"][0], query["postId"][0]) + response_text = await response.text() + soup = BeautifulSoup(response_text, "html.parser") + iframe = soup.find("iframe", id="control_frame") + if not iframe: + raise NazurinError("Failed to get real post ID") + src = urlparse(iframe.get("src")) + query = parse_qs(src.query) + return (query["blogId"][0], query["postId"][0]) diff --git a/nazurin/sites/lofter/config.py b/nazurin/sites/lofter/config.py index c6a6b797..746d6f1f 100644 --- a/nazurin/sites/lofter/config.py +++ b/nazurin/sites/lofter/config.py @@ -3,7 +3,6 @@ PRIORITY = 8 COLLECTION = "lofter" -with env.prefixed("LOFTER_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Lofter") - FILENAME: str = env.str("NAME", default="{id}_{index} - {nickName}({blogName})") +with env.prefixed("LOFTER_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Lofter") + FILENAME: str = env.str("NAME", default="{id}_{index} - {nickName}({blogName})") diff --git a/nazurin/sites/lofter/interface.py b/nazurin/sites/lofter/interface.py index e789fd07..8514fbd8 100644 --- a/nazurin/sites/lofter/interface.py +++ b/nazurin/sites/lofter/interface.py @@ -8,7 +8,7 @@ patterns = [ # https://username.lofter.com/post/1a2b3c4d_1a2b3c4d5 - r"(\w+)\.lofter\.com/post/(\w+)" + r"(\w+)\.lofter\.com/post/(\w+)", ] diff --git a/nazurin/sites/moebooru/__init__.py b/nazurin/sites/moebooru/__init__.py index 98d0cd41..4c9bedb7 100644 --- a/nazurin/sites/moebooru/__init__.py +++ b/nazurin/sites/moebooru/__init__.py @@ -1,7 +1,7 @@ """Moebooru site plugin.""" from .api import Moebooru -from .commands import * +from .commands import * # noqa: F403 from .config import PRIORITY from .interface import handle, patterns diff --git a/nazurin/sites/moebooru/api.py b/nazurin/sites/moebooru/api.py index dc54baf0..38cb0d44 100644 --- a/nazurin/sites/moebooru/api.py +++ b/nazurin/sites/moebooru/api.py @@ -1,12 +1,12 @@ import json import os -from datetime import datetime +from datetime import datetime, timezone from typing import List, Optional, Tuple from urllib.parse import unquote from aiohttp.client_exceptions import ClientResponseError from bs4 import BeautifulSoup -from pybooru import Moebooru as moebooru +from pybooru import Moebooru as MoebooruBase from nazurin.config import TEMP_DIR from nazurin.models import Caption, Illust, Image @@ -29,14 +29,13 @@ def site(self, site_url: Optional[str] = "yande.re"): @network_retry async def get_post(self, post_id: int): url = "https://" + self.url + "/post/show/" + str(post_id) - async with Request() as request: - async with request.get(url) as response: - try: - response.raise_for_status() - except ClientResponseError as err: - raise NazurinError(err) from None - response = await response.text() - soup = BeautifulSoup(response, "html.parser") + async with Request() as request, request.get(url) as response: + try: + response.raise_for_status() + except ClientResponseError as err: + raise NazurinError(err) from None + response_text = await response.text() + soup = BeautifulSoup(response_text, "html.parser") tag = soup.find(id="post-view").find(recursive=False) if tag.name == "script": content = str.strip(tag.string) @@ -61,24 +60,21 @@ async def view(self, post_id: int) -> Illust: caption = self.build_caption(post, tags) return Illust(post_id, imgs, caption, post) - def pool(self, pool_id: int, jpeg=False): - client = moebooru(self.site) + def pool(self, pool_id: int, *, jpeg=False): + client = MoebooruBase(self.site) info = client.pool_posts(id=pool_id) posts = info["posts"] imgs = [] for post in posts: - if not jpeg: - url = post["file_url"] - else: - url = post["jpeg_url"] + url = post["file_url"] if not jpeg else post["jpeg_url"] name, _ = self.parse_url(url) destination, filename = self.get_storage_dest(post, name) imgs.append(Image(filename, url, destination)) caption = Caption({"name": info["name"], "description": info["description"]}) return imgs, caption - async def download_pool(self, pool_id, jpeg=False): - imgs, caption = self.pool(pool_id, jpeg) + async def download_pool(self, pool_id, *, jpeg=False): + imgs, caption = self.pool(pool_id, jpeg=jpeg) pool_name = caption["name"] await ensure_existence_async(os.path.join(TEMP_DIR, pool_name)) for key, img in enumerate(imgs): @@ -102,7 +98,7 @@ def get_images(self, post) -> List[Image]: post["file_size"], post["width"], post["height"], - ) + ), ] return imgs @@ -112,7 +108,10 @@ def get_storage_dest(self, post: dict, filename: str) -> Tuple[str, str]: """ # `updated_at` is only available on yande.re, so we won't cover it here - created_at = datetime.fromtimestamp(post["created_at"]) + created_at = datetime.fromtimestamp( + post["created_at"], + tz=timezone.utc, + ) filename, extension = os.path.splitext(filename) # Site name in pascal case, i.e. Yandere, Konachan, Lolibooru site_name = snake_to_pascal(COLLECTIONS[self.url]) @@ -133,7 +132,7 @@ def build_caption(self, post, tags) -> Caption: """Build media caption from an post.""" title = post["tags"] source = post["source"] - tag_string = artists = str() + tag_string = artists = "" for tag, tag_type in tags.items(): if tag_type == "artist": artists += tag + " " @@ -148,7 +147,7 @@ def build_caption(self, post, tags) -> Caption: "source": source, "parent_id": post["parent_id"], "has_children": post["has_children"], - } + }, ) return caption diff --git a/nazurin/sites/moebooru/commands.py b/nazurin/sites/moebooru/commands.py index 797ee94b..8afc5f26 100644 --- a/nazurin/sites/moebooru/commands.py +++ b/nazurin/sites/moebooru/commands.py @@ -2,7 +2,7 @@ from aiogram.types import Message from nazurin import bot, dp -from nazurin.utils.exceptions import InvalidCommandUsage +from nazurin.utils.exceptions import InvalidCommandUsageError from .api import Moebooru @@ -10,13 +10,15 @@ @dp.message_handler( - Command(["yandere"]), args="POST_ID", description="View yandere post" + Command(["yandere"]), + args="POST_ID", + description="View yandere post", ) async def yandere_view(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("yandere") from e + raise InvalidCommandUsageError("yandere") from e if post_id < 0: await message.reply("Invalid post id!") return @@ -25,13 +27,15 @@ async def yandere_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["yandere_download"]), args="POST_ID", description="Download yandere post" + Command(["yandere_download"]), + args="POST_ID", + description="Download yandere post", ) async def yandere_download(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("yandere_download") from e + raise InvalidCommandUsageError("yandere_download") from e if post_id <= 0: await message.reply("Invalid post id!") return @@ -41,13 +45,15 @@ async def yandere_download(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["konachan"]), args="POST_ID", description="Download Konachan post" + Command(["konachan"]), + args="POST_ID", + description="Download Konachan post", ) async def konachan_view(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("konachan") from e + raise InvalidCommandUsageError("konachan") from e if post_id < 0: await message.reply("Invalid post id!") return @@ -56,13 +62,15 @@ async def konachan_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["konachan_download"]), args="POST_ID", description="Download Konachan post" + Command(["konachan_download"]), + args="POST_ID", + description="Download Konachan post", ) async def konachan_download(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("konachan_download") from e + raise InvalidCommandUsageError("konachan_download") from e if post_id <= 0: await message.reply("Invalid post id!") return diff --git a/nazurin/sites/moebooru/config.py b/nazurin/sites/moebooru/config.py index 41ff67ab..5e5529ce 100644 --- a/nazurin/sites/moebooru/config.py +++ b/nazurin/sites/moebooru/config.py @@ -7,7 +7,6 @@ "lolibooru.moe": "lolibooru", } -with env.prefixed("MOEBOORU_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="{site_name}") - FILENAME: str = env.str("NAME", default="{filename}") +with env.prefixed("MOEBOORU_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="{site_name}") + FILENAME: str = env.str("NAME", default="{filename}") diff --git a/nazurin/sites/moebooru/interface.py b/nazurin/sites/moebooru/interface.py index fca8c413..8833419c 100644 --- a/nazurin/sites/moebooru/interface.py +++ b/nazurin/sites/moebooru/interface.py @@ -30,6 +30,8 @@ async def handle(match: re.Match) -> HandlerResult: illust = await api.view(int(post_id)) document = Document( - id=illust.id, collection=COLLECTIONS[site_url], data=illust.metadata + id=illust.id, + collection=COLLECTIONS[site_url], + data=illust.metadata, ) return illust, document diff --git a/nazurin/sites/pixiv/__init__.py b/nazurin/sites/pixiv/__init__.py index 6897459d..7841b59f 100644 --- a/nazurin/sites/pixiv/__init__.py +++ b/nazurin/sites/pixiv/__init__.py @@ -1,7 +1,7 @@ """Pixiv site plugin.""" from .api import Pixiv -from .commands import * +from .commands import * # noqa: F403 from .config import PRIORITY from .interface import handle, patterns diff --git a/nazurin/sites/pixiv/api.py b/nazurin/sites/pixiv/api.py index 8f0873d8..c474be02 100644 --- a/nazurin/sites/pixiv/api.py +++ b/nazurin/sites/pixiv/api.py @@ -33,6 +33,7 @@ from .models import PixivIllust, PixivImage SANITY_LEVEL_LIMITED = "https://s.pximg.net/common/images/limit_sanity_level_360.png" +TOKEN_EXPIRATION_SECONDS = 3600 class Pixiv: @@ -53,7 +54,10 @@ def __init__(self): Pixiv.api.set_accept_language(TRANSLATION) async def require_auth(self): - if Pixiv.api.access_token and time.time() - Pixiv.updated_time < 3600: + if ( + Pixiv.api.access_token + and time.time() - Pixiv.updated_time < TOKEN_EXPIRATION_SECONDS + ): # Logged in, access_token not expired return if Pixiv.api.refresh_token: @@ -67,7 +71,9 @@ async def require_auth(self): Pixiv.api.access_token = tokens["access_token"] Pixiv.api.refresh_token = tokens["refresh_token"] Pixiv.updated_time = tokens["updated_time"] - if time.time() - Pixiv.updated_time >= 3600: # Token expired + if ( + time.time() - Pixiv.updated_time >= TOKEN_EXPIRATION_SECONDS + ): # Token expired await self.refresh_token() else: logger.info("Pixiv logged in through cached tokens") @@ -100,7 +106,7 @@ async def get_artwork(self, artwork_id: int): raise NazurinError("Artwork is private") return illust - async def view(self, artwork_id: int = None) -> Illust: + async def view(self, artwork_id: int) -> Illust: illust = await self.get_artwork(artwork_id) if illust.type == "ugoira": illust = await self.view_ugoira(illust) @@ -133,7 +139,9 @@ async def view_ugoira(self, illust) -> Ugoira: return Ugoira(illust.id, video, caption, illust, files) async def bookmark( - self, artwork_id: int, privacy: PixivPrivacy = PixivPrivacy.PUBLIC + self, + artwork_id: int, + privacy: PixivPrivacy = PixivPrivacy.PUBLIC, ): response = await self.call(Pixiv.illust_bookmark_add, artwork_id, privacy.value) if "error" in response: @@ -154,7 +162,7 @@ async def refresh_token(self): "access_token": Pixiv.api.access_token, "refresh_token": Pixiv.api.refresh_token, "updated_time": Pixiv.updated_time, - } + }, ) logger.info("Pixiv tokens updated") @@ -163,7 +171,7 @@ async def call(self, func: Callable, *args): await self.require_auth() response = await func(*args) if ( - "error" in response.keys() and "invalid_grant" in response.error.message + "error" in response and "invalid_grant" in response.error.message ): # Access token expired await self.refresh_token() response = await func(*args) @@ -200,7 +208,9 @@ def convert(config: File, output: File): logger.info("Calling FFmpeg with command: {}", cmd) try: output = subprocess.check_output( - args, stderr=subprocess.STDOUT, shell=False + args, + stderr=subprocess.STDOUT, + shell=False, ) except subprocess.CalledProcessError as error: logger.error( @@ -255,7 +265,7 @@ def get_images(self, illust) -> List[PixivImage]: thumbnail=self.get_thumbnail(url), width=width, height=height, - ) + ), ) # For multi-page illusts, # width & height will be the size of the first page, @@ -279,14 +289,14 @@ def get_images(self, illust) -> List[PixivImage]: thumbnail=self.get_thumbnail(url), width=width, height=height, - ) + ), ) return imgs @staticmethod def build_caption(illust) -> Caption: """Build media caption from an artwork.""" - tags = str() + tags = "" for tag in illust.tags: if TRANSLATION and tag.translated_name: tag_name = tag.translated_name @@ -301,7 +311,7 @@ def build_caption(illust) -> Caption: "total_bookmarks": illust.total_bookmarks, "url": "pixiv.net/i/" + str(illust.id), "bookmarked": illust.is_bookmarked, - } + }, ) return caption @@ -336,7 +346,7 @@ def get_thumbnail(url: str) -> str: thumbnail = pre + "_master1200.jpg" return thumbnail - async def auth(self, retry=True): + async def auth(self, *, retry=True): try: await Pixiv.api_auth() if not retry: @@ -351,12 +361,13 @@ async def auth(self, retry=True): if retry: random_ua = f"PixivAndroidApp/6.{random.randrange(0, 60)}.0" logger.info( - "Blocked by CloudFlare, retry with random UA: {}", random_ua + "Blocked by CloudFlare, retry with random UA: {}", + random_ua, ) Pixiv.api.additional_headers = {"User-Agent": random_ua} return await self.auth(retry=False) logger.error(error) raise NazurinError( - "Blocked by CloudFlare security check, please try again later." + "Blocked by CloudFlare security check, please try again later.", ) from None raise error diff --git a/nazurin/sites/pixiv/commands.py b/nazurin/sites/pixiv/commands.py index 064a7038..de6e3aff 100644 --- a/nazurin/sites/pixiv/commands.py +++ b/nazurin/sites/pixiv/commands.py @@ -4,7 +4,7 @@ from aiogram.types import Message from nazurin import bot, dp -from nazurin.utils.exceptions import InvalidCommandUsage +from nazurin.utils.exceptions import InvalidCommandUsageError from .api import Pixiv from .config import PixivPrivacy @@ -13,13 +13,15 @@ @dp.message_handler( - Command(["pixiv"]), args="ARTWORK_ID", description="View Pixiv artwork" + Command(["pixiv"]), + args="ARTWORK_ID", + description="View Pixiv artwork", ) async def pixiv_view(message: Message, command: Command.CommandObj): try: artwork_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("pixiv") from e + raise InvalidCommandUsageError("pixiv") from e if artwork_id < 0: await message.reply("Invalid artwork id!") return @@ -28,13 +30,15 @@ async def pixiv_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["pixiv_download"]), args="ARTWORK_ID", description="Download Pixiv artwork" + Command(["pixiv_download"]), + args="ARTWORK_ID", + description="Download Pixiv artwork", ) async def pixiv_download(message: Message, command: Command.CommandObj): try: artwork_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("pixiv_download") from e + raise InvalidCommandUsageError("pixiv_download") from e if artwork_id < 0: await message.reply("Invalid artwork id!") return @@ -52,7 +56,7 @@ async def pixiv_bookmark(message: Message, command: Command.CommandObj): try: artwork_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("pixiv_bookmark") from e + raise InvalidCommandUsageError("pixiv_bookmark") from e if artwork_id < 0: await message.reply("Invalid artwork id!") return diff --git a/nazurin/sites/pixiv/config.py b/nazurin/sites/pixiv/config.py index 5c179ebb..c6edf40f 100644 --- a/nazurin/sites/pixiv/config.py +++ b/nazurin/sites/pixiv/config.py @@ -28,7 +28,8 @@ class PixivPrivacy(Enum): with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Pixiv") FILENAME: str = env.str( - "NAME", default="{filename} - {title} - {user[name]}({user[id]})" + "NAME", + default="{filename} - {title} - {user[name]}({user[id]})", ) HEADERS = {"Referer": "https://app-api.pixiv.net/"} diff --git a/nazurin/sites/twitter/__init__.py b/nazurin/sites/twitter/__init__.py index c3d58a4f..0ee925e4 100644 --- a/nazurin/sites/twitter/__init__.py +++ b/nazurin/sites/twitter/__init__.py @@ -1,7 +1,7 @@ """Twitter site plugin.""" from .api import Twitter -from .commands import * +from .commands import * # noqa: F403 from .config import PRIORITY from .interface import handle, patterns diff --git a/nazurin/sites/twitter/api/base.py b/nazurin/sites/twitter/api/base.py index 8d29b952..de79b748 100644 --- a/nazurin/sites/twitter/api/base.py +++ b/nazurin/sites/twitter/api/base.py @@ -14,11 +14,13 @@ class BaseAPI: def build_caption(tweet) -> Caption: return Caption( { - "url": f"https://twitter.com/{tweet['user']['screen_name']}" - + f"/status/{tweet['id_str']}", + "url": ( + f"https://twitter.com/{tweet['user']['screen_name']}" + f"/status/{tweet['id_str']}" + ), "author": f"{tweet['user']['name']} #{tweet['user']['screen_name']}", "text": tweet["text"], - } + }, ) @staticmethod @@ -69,7 +71,7 @@ async def get_best_video(tweet: dict, variants: list) -> Ugoira: if variant["content_type"] != "video/mp4": continue # https://video.twimg.com/amplify_video/1625137841473982464/vid/720x954/YzLr5Rw4xODqTpkm.mp4?tag=16 - bitrate = variant["bitrate"] if "bitrate" in variant else 0 + bitrate = variant.get("bitrate", 0) if bitrate > max_bitrate: max_bitrate = variant["bitrate"] best_variant = variant["url"] diff --git a/nazurin/sites/twitter/api/syndication.py b/nazurin/sites/twitter/api/syndication.py index ca51beeb..204e0e92 100644 --- a/nazurin/sites/twitter/api/syndication.py +++ b/nazurin/sites/twitter/api/syndication.py @@ -1,3 +1,4 @@ +from http import HTTPStatus from typing import List from nazurin.models import Illust, Image, Ugoira @@ -12,24 +13,27 @@ class SyndicationAPI(BaseAPI): """Public API from publish.twitter.com""" + API_URL = "https://cdn.syndication.twimg.com/tweet-result" + @network_retry async def get_tweet(self, status_id: int): """Get a tweet from API.""" logger.info("Fetching tweet {} from syndication API", status_id) - API_URL = "https://cdn.syndication.twimg.com/tweet-result" params = { "features": "tfw_tweet_edit_backend:on", "id": str(status_id), "lang": "en", } - async with Request() as request: - async with request.get(API_URL, params=params) as response: - if response.status == 404: - raise NazurinError("Tweet not found or unavailable.") - response.raise_for_status() - tweet = await response.json() - del tweet["__typename"] - return tweet + async with Request() as request, request.get( + self.API_URL, + params=params, + ) as response: + if response.status == HTTPStatus.NOT_FOUND: + raise NazurinError("Tweet not found or unavailable.") + response.raise_for_status() + tweet = await response.json() + del tweet["__typename"] + return tweet async def fetch(self, status_id: int) -> Illust: """Fetch & return tweet images and information.""" diff --git a/nazurin/sites/twitter/api/web.py b/nazurin/sites/twitter/api/web.py index 4b75de19..de21b10d 100644 --- a/nazurin/sites/twitter/api/web.py +++ b/nazurin/sites/twitter/api/web.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import json import secrets -from datetime import datetime +from datetime import datetime, timezone +from http import HTTPStatus from http.cookies import SimpleCookie -from typing import List +from typing import ClassVar from nazurin.models import Illust, Image from nazurin.utils.decorators import Cache, network_retry @@ -43,9 +46,16 @@ class TweetDetailAPI: LOGGED_IN = "q94uRCEn65LZThakYcPT6g/TweetDetail" +ERROR_MESSAGES = { + "NsfwLoggedOut": "NSFW tweet, please log in", + "Protected": "Protected tweet, you may try logging in if you have access", + "Suspended": "This account has been suspended", +} + + class WebAPI(BaseAPI): auth_token = AUTH_TOKEN - headers = { + headers: ClassVar[dict[str, str]] = { "Authorization": AuthorizationToken.GUEST, "Origin": "https://twitter.com", "Referer": "https://twitter.com", @@ -53,7 +63,7 @@ class WebAPI(BaseAPI): "x-twitter-client-language": "en", "x-twitter-active-user": "yes", } - variables = { + variables: ClassVar[dict[str, bool]] = { "with_rux_injections": False, "includePromotedContent": False, "withCommunity": True, @@ -65,7 +75,7 @@ class WebAPI(BaseAPI): "withVoice": True, "withV2Timeline": True, } - features = { + features: ClassVar[dict[str, bool]] = { "blue_business_profile_image_shape_enabled": False, "rweb_lists_timeline_redesign_enabled": True, "responsive_web_graphql_exclude_directive_enabled": True, @@ -113,7 +123,7 @@ async def fetch(self, status_id: int) -> Illust: if "extended_entities" not in tweet: raise NazurinError("No photo found.") media = tweet["extended_entities"]["media"] - imgs: List[Image] = [] + imgs: list[Image] = [] for medium in media: if medium["type"] == "photo": index = len(imgs) @@ -127,7 +137,7 @@ async def fetch(self, status_id: int) -> Illust: "height": original_info["height"], }, index, - ) + ), ) else: # video or animated_gif @@ -149,8 +159,8 @@ async def tweet_detail(self, tweet_id: str): AuthorizationToken.LOGGED_IN if AUTH_TOKEN else AuthorizationToken.GUEST - ) - } + ), + }, ) variables = WebAPI.variables variables.update({"focalTweetId": tweet_id}) @@ -206,41 +216,44 @@ async def _request(self, method, url, headers=None, **kwargs): headers = {} headers.update(WebAPI.headers) - async with Request(headers=headers, cookies=WebAPI.cookies) as request: - async with request.request(method, url, **kwargs) as response: - if not response.ok: - result = await response.text() - logger.error("Web API Error: {}, {}", response.status, result) - if response.status == 401: - raise NazurinError( - f"Failed to authenticate Twitter web API: {result}, " - "try updating auth token." + async with Request( + headers=headers, + cookies=WebAPI.cookies, + ) as request, request.request(method, url, **kwargs) as response: + if not response.ok: + result = await response.text() + logger.error("Web API Error: {}, {}", response.status, result) + if response.status == HTTPStatus.UNAUTHORIZED: + raise NazurinError( + f"Failed to authenticate Twitter web API: {result}, " + "try updating auth token.", + ) + if response.status == HTTPStatus.TOO_MANY_REQUESTS: + headers = response.headers + detail = "" + if ( + Headers.RATE_LIMIT_LIMIT in headers + and Headers.RATE_LIMIT_RESET in headers + ): + rate_limit = int(headers[Headers.RATE_LIMIT_LIMIT]) + reset_time = int(headers[Headers.RATE_LIMIT_RESET]) + logger.error( + "Rate limited, limit: {}, reset: {}", + rate_limit, + reset_time, ) - if response.status == 429: - headers = response.headers - detail = "" - if ( - Headers.RATE_LIMIT_LIMIT in headers - and Headers.RATE_LIMIT_RESET in headers - ): - rate_limit = int(headers[Headers.RATE_LIMIT_LIMIT]) - reset_time = int(headers[Headers.RATE_LIMIT_RESET]) - logger.error( - "Rate limited, limit: {}, reset: {}", - rate_limit, - reset_time, - ) - reset_time = datetime.fromtimestamp(reset_time) - detail = ( - f"Rate limit: {rate_limit}, Reset time: {reset_time}" - ) - raise NazurinError( - "Hit API rate limit, please try again later. " + detail + reset_time = datetime.fromtimestamp( + reset_time, + tz=timezone.utc, ) - raise NazurinError(f"Twitter web API error: {result}") - result = await response.json() - self._update_cookies(response.cookies) - return result + detail = f"Rate limit: {rate_limit}, Reset time: {reset_time}" + raise NazurinError( + "Hit API rate limit, please try again later. " + detail, + ) + raise NazurinError(f"Twitter web API error: {result}") + result = await response.json() + self._update_cookies(response.cookies) + return result def _update_cookies(self, cookies: SimpleCookie): WebAPI.cookies.update(cookies) @@ -267,7 +280,7 @@ def _process_response(self, response: dict, tweet_id: str): if "errors" in response: logger.error(response) raise NazurinError( - "\n".join([error["message"] for error in response["errors"]]) + "\n".join([error["message"] for error in response["errors"]]), ) instructions = response["data"]["threaded_conversation_with_injections_v2"][ @@ -288,7 +301,7 @@ def _process_response(self, response: dict, tweet_id: str): raise NazurinError( "Tweet result is empty, maybe it's a sensitive tweet " "or the author limited visibility, " - "you may try setting an AUTH_TOKEN." + "you may try setting an AUTH_TOKEN.", ) tweet = tweet["result"] break @@ -307,7 +320,7 @@ def _process_response(self, response: dict, tweet_id: str): text = tombstone["text"] if text.startswith("Age-restricted"): raise NazurinError( - "Age-restricted adult content. Please set Twitter auth token." + "Age-restricted adult content. Please set Twitter auth token.", ) raise NazurinError(text) @@ -338,7 +351,7 @@ def normalize_tweet(data: dict): "created_at": fromasctimeformat(tweet["created_at"]).isoformat(), "user": WebAPI.normalize_user(data["core"]["user_results"]["result"]), "text": tweet["full_text"], - } + }, ) del tweet["full_text"] return tweet @@ -355,17 +368,12 @@ def normalize_user(data: dict): "id_str": data["rest_id"], "created_at": fromasctimeformat(user["created_at"]).isoformat(), "is_blue_verified": data["is_blue_verified"], - } + }, ) return user @staticmethod def error_message_by_reason(reason: str): - MESSAGES = { - "NsfwLoggedOut": "NSFW tweet, please log in", - "Protected": "Protected tweet, you may try logging in if you have access", - "Suspended": "This account has been suspended", - } - if reason in MESSAGES: - return MESSAGES[reason] + if reason in ERROR_MESSAGES: + return ERROR_MESSAGES[reason] return reason diff --git a/nazurin/sites/twitter/commands.py b/nazurin/sites/twitter/commands.py index 85e0656d..8fd6e6f7 100644 --- a/nazurin/sites/twitter/commands.py +++ b/nazurin/sites/twitter/commands.py @@ -2,7 +2,7 @@ from aiogram.types import Message from nazurin import bot, dp -from nazurin.utils.exceptions import InvalidCommandUsage +from nazurin.utils.exceptions import InvalidCommandUsageError from .api import Twitter @@ -14,7 +14,7 @@ async def twitter_view(message: Message, command: Command.CommandObj): try: status_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("twitter") from e + raise InvalidCommandUsageError("twitter") from e if status_id < 0: await message.reply("Invalid status id.") return @@ -23,13 +23,15 @@ async def twitter_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["twitter_download"]), args="STATUS_ID", description="Download tweet" + Command(["twitter_download"]), + args="STATUS_ID", + description="Download tweet", ) async def twitter_download(message: Message, command: Command.CommandObj): try: status_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("twitter_download") from e + raise InvalidCommandUsageError("twitter_download") from e if status_id < 0: await message.reply("Invalid status id.") return diff --git a/nazurin/sites/twitter/config.py b/nazurin/sites/twitter/config.py index 20a628f7..de4f0206 100644 --- a/nazurin/sites/twitter/config.py +++ b/nazurin/sites/twitter/config.py @@ -13,7 +13,10 @@ class TwitterAPI(Enum): with env.prefixed("TWITTER_"): API: TwitterAPI = env.enum( - "API", type=TwitterAPI, default=TwitterAPI.WEB.value, ignore_case=True + "API", + type=TwitterAPI, + default=TwitterAPI.WEB.value, + ignore_case=True, ) # Auth token for web API AUTH_TOKEN: str = env.str("AUTH_TOKEN", default=None) @@ -21,5 +24,6 @@ class TwitterAPI(Enum): with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Twitter") FILENAME: str = env.str( - "NAME", default="{id_str}_{index} - {user[name]}({user[id_str]})" + "NAME", + default="{id_str}_{index} - {user[name]}({user[id_str]})", ) diff --git a/nazurin/sites/twitter/interface.py b/nazurin/sites/twitter/interface.py index 3ee1089f..579057cf 100644 --- a/nazurin/sites/twitter/interface.py +++ b/nazurin/sites/twitter/interface.py @@ -11,7 +11,7 @@ # https://twitter.com/abcdefg/status/1234567890123456789 # https://www.twitter.com/abcdefg/status/1234567890123456789 # https://mobile.twitter.com/abcdefg/status/1234567890123456789 - r"(?:mobile\.|www\.)?(?:twitter|x)\.com/[^.]+/status/(\d+)" + r"(?:mobile\.|www\.)?(?:twitter|x)\.com/[^.]+/status/(\d+)", ] diff --git a/nazurin/sites/wallhaven/api.py b/nazurin/sites/wallhaven/api.py index 93029343..de209f39 100644 --- a/nazurin/sites/wallhaven/api.py +++ b/nazurin/sites/wallhaven/api.py @@ -1,5 +1,6 @@ import os from datetime import datetime +from http import HTTPStatus from typing import List, Tuple from nazurin.models import Caption, Illust, Image @@ -17,20 +18,21 @@ async def get_wallpaper(self, wallpaper_id: str): api = "https://wallhaven.cc/api/v1/w/" + wallpaper_id if API_KEY: api += "?apikey=" + API_KEY - async with Request() as request: - async with request.get(api) as response: - if response.status == 404: - raise NazurinError("Wallpaper doesn't exist.") - if response.status == 401: - raise NazurinError( + async with Request() as request, request.get(api) as response: + if response.status == HTTPStatus.NOT_FOUND: + raise NazurinError("Wallpaper doesn't exist.") + if response.status == HTTPStatus.UNAUTHORIZED: + raise NazurinError( + ( "You need to log in to view this wallpaper. " - + "Please ensure that you have set a valid API key." - ) - response.raise_for_status() - wallpaper = await response.json() - if "error" in wallpaper: - raise NazurinError(wallpaper["error"]) - return wallpaper["data"] + "Please ensure that you have set a valid API key." + ), + ) + response.raise_for_status() + wallpaper = await response.json() + if "error" in wallpaper: + raise NazurinError(wallpaper["error"]) + return wallpaper["data"] async def fetch(self, wallpaper_id: str) -> Illust: """Fetch & return wallpaper image and information.""" @@ -53,7 +55,7 @@ def get_images(wallpaper) -> List[Image]: wallpaper["file_size"], wallpaper["dimension_x"], wallpaper["dimension_y"], - ) + ), ] @staticmethod @@ -71,9 +73,9 @@ def get_storage_dest(wallpaper: dict) -> Tuple[str, str]: @staticmethod def build_caption(wallpaper) -> Caption: - tags = str() + tags = "" for tag in wallpaper["tags"]: tags += "#" + tag["name"].strip().replace(" ", "_") + " " return Caption( - {"url": wallpaper["url"], "source": wallpaper["source"], "tags": tags} + {"url": wallpaper["url"], "source": wallpaper["source"], "tags": tags}, ) diff --git a/nazurin/sites/wallhaven/interface.py b/nazurin/sites/wallhaven/interface.py index bf90e4e2..df0aedb9 100644 --- a/nazurin/sites/wallhaven/interface.py +++ b/nazurin/sites/wallhaven/interface.py @@ -9,7 +9,7 @@ patterns = [ # http://whvn.cc/94x38z # https://wallhaven.cc/w/94x38z - r"(?:wallhaven|whvn)\.cc\/(?:w\/)?([\w]+)" + r"(?:wallhaven|whvn)\.cc\/(?:w\/)?([\w]+)", ] diff --git a/nazurin/sites/weibo/api.py b/nazurin/sites/weibo/api.py index 36936655..62b71c4f 100644 --- a/nazurin/sites/weibo/api.py +++ b/nazurin/sites/weibo/api.py @@ -18,12 +18,11 @@ class Weibo: async def get_post(self, post_id: str): """Fetch a post.""" api = f"https://m.weibo.cn/detail/{post_id}" - async with Request() as request: - async with request.get(api) as response: - response.raise_for_status() - html = await response.text() - post = self.parse_html(html) - return post + async with Request() as request, request.get(api) as response: + response.raise_for_status() + html = await response.text() + post = self.parse_html(html) + return post async def fetch(self, post_id: str) -> WeiboIllust: post = await self.get_post(post_id) @@ -55,7 +54,7 @@ def get_images(self, post) -> List[WeiboImage]: width=width, height=height, referer=f"https://m.weibo.cn/detail/{post['mid']}", - ) + ), ) return imgs @@ -80,7 +79,7 @@ def get_storage_dest(post: dict, pic: dict, index: int) -> Tuple[str, str]: def build_caption(self, post) -> Caption: user = post["user"] tags = self.get_tags(post) - tag_string = str() + tag_string = "" for tag in tags: tag_string += "#" + tag + " " return Caption( @@ -90,7 +89,7 @@ def build_caption(self, post) -> Caption: "desktop_url": f"https://weibo.com/{user['id']}/{post['bid']}", "mobile_url": f"https://m.weibo.cn/detail/{post['mid']}", "tags": tag_string, - } + }, ) @staticmethod diff --git a/nazurin/sites/weibo/config.py b/nazurin/sites/weibo/config.py index ef712a29..c986d72b 100644 --- a/nazurin/sites/weibo/config.py +++ b/nazurin/sites/weibo/config.py @@ -3,9 +3,9 @@ PRIORITY = 5 COLLECTION = "weibo" -with env.prefixed("WEIBO_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Weibo") - FILENAME: str = env.str( - "NAME", default="{mid}_{index} - {user[screen_name]}({user[id]})" - ) +with env.prefixed("WEIBO_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Weibo") + FILENAME: str = env.str( + "NAME", + default="{mid}_{index} - {user[screen_name]}({user[id]})", + ) diff --git a/nazurin/sites/zerochan/__init__.py b/nazurin/sites/zerochan/__init__.py index 4e4d7520..ec9265d1 100644 --- a/nazurin/sites/zerochan/__init__.py +++ b/nazurin/sites/zerochan/__init__.py @@ -1,7 +1,7 @@ """Zerochan site plugin.""" from .api import Zerochan -from .commands import * +from .commands import * # noqa: F403 from .config import PRIORITY from .interface import handle, patterns diff --git a/nazurin/sites/zerochan/api.py b/nazurin/sites/zerochan/api.py index fb289c16..aab4c4ca 100644 --- a/nazurin/sites/zerochan/api.py +++ b/nazurin/sites/zerochan/api.py @@ -16,18 +16,17 @@ class Zerochan: @network_retry async def get_post(self, post_id: int): - async with Request() as request: - async with request.get( - "https://www.zerochan.net/" + str(post_id) - ) as response: - response.raise_for_status() + async with Request() as request, request.get( + "https://www.zerochan.net/" + str(post_id), + ) as response: + response.raise_for_status() - # Override post_id if there's a redirection TODO: Check - if response.history: - post_id = response.url.path[1:] - response = await response.text() + # Override post_id if there's a redirection TODO: Check + if response.history: + post_id = response.url.path[1:] + response_text = await response.text() - soup = BeautifulSoup(response, "html.parser") + soup = BeautifulSoup(response_text, "html.parser") info = soup.find("script", {"type": "application/ld+json"}).contents info = json.loads("".join(info).replace("\\'", "'")) @@ -73,7 +72,7 @@ def get_images(post) -> List[Image]: post["file_size"], int(post["image_width"]), int(post["image_height"]), - ) + ), ] @staticmethod @@ -97,7 +96,7 @@ def get_storage_dest(post: dict) -> Tuple[str, str]: @staticmethod def build_caption(post) -> Caption: """Build media caption from an post.""" - tag_string = artists = source = str() + tag_string = artists = source = "" for tag, tag_type in post["tags"].items(): if tag_type == "Mangaka": artists += tag + " " @@ -112,6 +111,6 @@ def build_caption(post) -> Caption: "url": "https://www.zerochan.net/" + str(post["id"]), "source": source, "tags": tag_string, - } + }, ) return caption diff --git a/nazurin/sites/zerochan/commands.py b/nazurin/sites/zerochan/commands.py index bf89cf1e..b372e44a 100644 --- a/nazurin/sites/zerochan/commands.py +++ b/nazurin/sites/zerochan/commands.py @@ -2,7 +2,7 @@ from aiogram.types import Message from nazurin import bot, dp -from nazurin.utils.exceptions import InvalidCommandUsage +from nazurin.utils.exceptions import InvalidCommandUsageError from .api import Zerochan @@ -10,13 +10,15 @@ @dp.message_handler( - Command(["zerochan"]), args="POST_ID", description="View Zerochan post" + Command(["zerochan"]), + args="POST_ID", + description="View Zerochan post", ) async def zerochan_view(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("zerochan") from e + raise InvalidCommandUsageError("zerochan") from e if post_id < 0: await message.reply("Invalid post id!") return @@ -25,13 +27,15 @@ async def zerochan_view(message: Message, command: Command.CommandObj): @dp.message_handler( - Command(["zerochan_download"]), args="POST_ID", description="Download Zerochan post" + Command(["zerochan_download"]), + args="POST_ID", + description="Download Zerochan post", ) async def zerochan_download(message: Message, command: Command.CommandObj): try: post_id = int(command.args) except (IndexError, ValueError, TypeError) as e: - raise InvalidCommandUsage("zerochan_download") from e + raise InvalidCommandUsageError("zerochan_download") from e if post_id <= 0: await message.reply("Invalid post id!") return diff --git a/nazurin/sites/zerochan/config.py b/nazurin/sites/zerochan/config.py index d74c4bb6..a20008db 100644 --- a/nazurin/sites/zerochan/config.py +++ b/nazurin/sites/zerochan/config.py @@ -3,7 +3,6 @@ PRIORITY = 9 COLLECTION = "zerochan" -with env.prefixed("ZEROCHAN_"): - with env.prefixed("FILE_"): - DESTINATION: str = env.str("PATH", default="Zerochan") - FILENAME: str = env.str("NAME", default="{id} - {name}") +with env.prefixed("ZEROCHAN_"), env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Zerochan") + FILENAME: str = env.str("NAME", default="{id} - {name}") diff --git a/nazurin/storage/__init__.py b/nazurin/storage/__init__.py index cbec2d6b..2e458251 100644 --- a/nazurin/storage/__init__.py +++ b/nazurin/storage/__init__.py @@ -2,6 +2,7 @@ import asyncio import importlib +from typing import ClassVar, List from nazurin.config import STORAGE from nazurin.models import Illust @@ -11,7 +12,7 @@ class Storage: """Storage manager.""" - disks = [] + disks: ClassVar[List[object]] = [] def load(self): """Dynamically load all storage drivers.""" @@ -21,8 +22,6 @@ def load(self): logger.info("Loaded {} storage(s), using: {}", len(self.disks), STORAGE) async def store(self, illust: Illust): - tasks = [] - for disk in self.disks: - tasks.append(disk.store(illust.all_files)) + tasks = [disk.store(illust.all_files) for disk in self.disks] await asyncio.gather(*tasks) logger.info("Storage completed") diff --git a/nazurin/storage/googledrive.py b/nazurin/storage/googledrive.py index 05b550bd..42b4eeae 100644 --- a/nazurin/storage/googledrive.py +++ b/nazurin/storage/googledrive.py @@ -1,7 +1,7 @@ import asyncio import json from pathlib import PurePath -from typing import Awaitable, Callable, List +from typing import Awaitable, Callable, List, Optional from oauth2client.service_account import ServiceAccountCredentials from pydrive2.auth import GoogleAuth @@ -17,7 +17,8 @@ GD_FOLDER = env.str("GD_FOLDER") GD_CREDENTIALS = env.str( - "GD_CREDENTIALS", default=env.str("GOOGLE_APPLICATION_CREDENTIALS") + "GD_CREDENTIALS", + default=env.str("GOOGLE_APPLICATION_CREDENTIALS"), ) FOLDER_MIME = "application/vnd.google-apps.folder" @@ -27,7 +28,7 @@ class GoogleDrive: drive = GDrive() create_file: Callable[[dict], Awaitable[GoogleDriveFile]] = async_wrap( - drive.CreateFile + drive.CreateFile, ) def __init__(self): @@ -43,18 +44,20 @@ def auth(): if GD_CREDENTIALS.startswith("{"): credentials = json.loads(GD_CREDENTIALS) gauth.credentials = ServiceAccountCredentials.from_json_keyfile_dict( - credentials, scope + credentials, + scope, ) else: gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name( - GD_CREDENTIALS, scope + GD_CREDENTIALS, + scope, ) else: raise NazurinError("Credentials not found for Google Drive storage.") GoogleDrive.drive.auth = gauth @staticmethod - async def upload(file: File, folders: dict = None): + async def upload(file: File, folders: Optional[dict] = None): # Compute relative path to STORAGE_DIR, which is GD_FOLDER path = file.destination.relative_to(STORAGE_DIR).as_posix() parent = folders[path] if folders else await GoogleDrive.create_folders(path) @@ -83,7 +86,7 @@ async def store(self, files: List[File]): @staticmethod @Cache.lru() @async_wrap - def find_folder(name: str, parent: str = None) -> str: + def find_folder(name: str, parent: Optional[str] = None) -> str: query = { "q": f"mimeType='{FOLDER_MIME}' and " f"title='{name}' and " @@ -96,7 +99,7 @@ def find_folder(name: str, parent: str = None) -> str: return result[0].get("id") @staticmethod - async def create_folder(name: str, parent: str = None) -> str: + async def create_folder(name: str, parent: Optional[str] = None) -> str: metadata = { "title": name, "mimeType": FOLDER_MIME, @@ -107,7 +110,7 @@ async def create_folder(name: str, parent: str = None) -> str: return folder.get("id") @staticmethod - async def create_folders(path: str, parent: str = None) -> str: + async def create_folders(path: str, parent: Optional[str] = None) -> str: """ Create folders recursively. diff --git a/nazurin/storage/local.py b/nazurin/storage/local.py index 5d9c9faf..24f3031a 100644 --- a/nazurin/storage/local.py +++ b/nazurin/storage/local.py @@ -17,7 +17,8 @@ def __init__(self): @async_wrap def move_file(file: File): shutil.copyfile( - file.path, os.path.join(os.path.join(DATA_DIR, file.destination), file.name) + file.path, + os.path.join(os.path.join(DATA_DIR, file.destination), file.name), ) async def store(self, files: List[File]): diff --git a/nazurin/storage/mega.py b/nazurin/storage/mega.py index f9e2a672..035f5c3a 100644 --- a/nazurin/storage/mega.py +++ b/nazurin/storage/mega.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- import asyncio -from typing import List +from typing import List, Optional -from mega import Mega as mega +from mega import Mega as MegaBase from mega.errors import RequestError from nazurin.config import MAX_PARALLEL_UPLOAD, NAZURIN_DATA, env @@ -19,7 +18,7 @@ class Mega: - api = mega() + api = MegaBase() db = Database().driver() collection = db.collection(NAZURIN_DATA) document = collection.document(MEGA_DOCUMENT) @@ -30,7 +29,7 @@ class Mega: create_folder = async_wrap(api.create_folder) @network_retry - async def login(self, initialize=False): + async def login(self, *, initialize=False): await Mega.api_login(MEGA_USER, MEGA_PASS) if initialize: await Mega.collection.insert( @@ -47,7 +46,7 @@ async def login(self, initialize=False): "sid": Mega.api.sid, "master_key": list(Mega.api.master_key), "root_id": Mega.api.root_id, - } + }, ) logger.info("MEGA tokens cached") @@ -63,7 +62,13 @@ async def require_auth(self): await self.login(initialize=True) @network_retry - async def upload(self, file: File, folders: dict = None, retry: bool = False): + async def upload( + self, + file: File, + folders: Optional[dict] = None, + *, + retry: bool = False, + ): path = file.destination.as_posix() try: destination = ( diff --git a/nazurin/storage/onedrive.py b/nazurin/storage/onedrive.py index 5dbd0570..5db89ce6 100644 --- a/nazurin/storage/onedrive.py +++ b/nazurin/storage/onedrive.py @@ -20,7 +20,10 @@ OD_SECRET = env.str("OD_SECRET") OD_RF_TOKEN = env.str("OD_RF_TOKEN") # Refresh token for first-time auth OD_DOCUMENT = "onedrive" + BASE_URL = "https://graph.microsoft.com/v1.0" +# Must be a multiple of 320 KB +UPLOAD_CHUNK_SIZE = 16 * 320 * 1024 # 5MB class OneDrive: @@ -47,7 +50,7 @@ async def upload(self, file: File): body = { "item": { "@microsoft.graph.conflictBehavior": "replace", - } + }, } path = self.encode_path(pathlib.Path(file.destination, file.name)) create_session_url = ( @@ -138,7 +141,7 @@ async def require_auth(self): await self.auth(initialize=True) @network_retry - async def auth(self, initialize=False): + async def auth(self, *, initialize=False): # https://docs.microsoft.com/zh-cn/azure/active-directory/develop/v2-oauth2-auth-code-flow url = "https://login.microsoftonline.com/common/oauth2/v2.0/token" data = { @@ -147,17 +150,16 @@ async def auth(self, initialize=False): "refresh_token": self.refresh_token or OD_RF_TOKEN, "grant_type": "refresh_token", } - async with Request() as request: - async with request.post(url, data=data) as response: - response = await response.json() - if "error" in response: - logger.error(response) - raise NazurinError( - f"OneDrive authorization error: {response['error_description']}" - ) - self.access_token = response["access_token"] - self.refresh_token = response["refresh_token"] - self.expires_at = time.time() + response["expires_in"] + async with Request() as request, request.post(url, data=data) as response: + response_json = await response.json() + if "error" in response_json: + logger.error(response_json) + raise NazurinError( + f"OneDrive authorization error: {response_json['error_description']}", + ) + self.access_token = response_json["access_token"] + self.refresh_token = response_json["refresh_token"] + self.expires_at = time.time() + response_json["expires_in"] credentials = { "access_token": self.access_token, "refresh_token": self.refresh_token, @@ -186,14 +188,17 @@ async def _request(self, method, url, headers=None, **kwargs): # make a request with access token _headers = self.with_credentials(headers) _headers["Content-Type"] = "application/json" - async with Request(headers=_headers) as session: - async with session.request(method, url, **kwargs) as response: - if not response.ok: - logger.error(await response.text()) - response.raise_for_status() - if "application/json" in response.headers["Content-Type"]: - return await response.json() - return await response.text() + async with Request(headers=_headers) as session, session.request( + method, + url, + **kwargs, + ) as response: + if not response.ok: + logger.error(await response.text()) + response.raise_for_status() + if "application/json" in response.headers["Content-Type"]: + return await response.json() + return await response.text() async def stream_upload(self, file: File, url: str): @network_retry @@ -203,35 +208,35 @@ async def upload_chunk(url: str, chunk: bytes): logger.error(await response.text()) response.raise_for_status() - # Must be a multiple of 320 KB - CHUNK_SIZE = 16 * 320 * 1024 # 5MB headers = self.with_credentials() range_start = 0 total_size = await file.size() - total_size_str = naturalsize(total_size, True) + total_size_str = naturalsize(total_size, binary=True) logger.info( - "[File {}] Start upload, total size: {}...", file.name, total_size_str + "[File {}] Start upload, total size: {}...", + file.name, + total_size_str, ) async with Request(headers=headers) as session: - async for chunk in read_by_chunks(file.path, CHUNK_SIZE): + async for chunk in read_by_chunks(file.path, UPLOAD_CHUNK_SIZE): content_length = len(chunk) range_end = range_start + content_length - 1 session.headers.update({"Content-Length": str(content_length)}) session.headers.update( - {"Content-Range": f"bytes {range_start}-{range_end}/{total_size}"} + {"Content-Range": f"bytes {range_start}-{range_end}/{total_size}"}, ) await upload_chunk(url, chunk) range_start += content_length logger.info( "[File {}] Uploaded {} / {}", file.name, - naturalsize(range_start, True), + naturalsize(range_start, binary=True), total_size_str, ) logger.info("[File {}] Upload completed", file.name) - def with_credentials(self, headers: dict = None) -> dict: + def with_credentials(self, headers: Optional[dict] = None) -> dict: """ Add credentials to the request header. """ diff --git a/nazurin/storage/telegram.py b/nazurin/storage/telegram.py index aadeac3e..37775d7b 100644 --- a/nazurin/storage/telegram.py +++ b/nazurin/storage/telegram.py @@ -22,7 +22,7 @@ async def store(self, files: List[File]): logger.warning( "File {} exceeds size limit ({}) and won't be save to Telegram", file.name, - naturalsize(size, True), + naturalsize(size, binary=True), ) continue tasks.append(bot.send_doc(file, chat_id=ALBUM_ID)) diff --git a/nazurin/tests/__init__.py b/nazurin/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nazurin/tests/test_commands_manager.py b/nazurin/tests/test_commands_manager.py index f7761208..9e927bad 100644 --- a/nazurin/tests/test_commands_manager.py +++ b/nazurin/tests/test_commands_manager.py @@ -29,7 +29,8 @@ def test_resolve_names(self): self.assertEqual(self.manager.resolve_names(Command(names)), names) self.assertEqual(self.manager.resolve_names(commands=names), names) self.assertEqual( - self.manager.resolve_names(Command(["second"]), commands=["first"]), names + self.manager.resolve_names(Command(["second"]), commands=["first"]), + names, ) def test_command_list(self): @@ -63,7 +64,7 @@ def test_command_help(self): Usage: /alternative, /first First Command Help Text - """ + """, ), ) self.assertEqual( @@ -72,7 +73,7 @@ def test_command_help(self): """ Second Command Usage: /second ARG - """ + """, ), ) self.assertIsNone(self.manager.help("third")) diff --git a/nazurin/utils/decorators.py b/nazurin/utils/decorators.py index 2780e491..56fcd59e 100644 --- a/nazurin/utils/decorators.py +++ b/nazurin/utils/decorators.py @@ -1,6 +1,7 @@ import asyncio import functools from functools import partial, wraps +from typing import Callable, ClassVar, List import tenacity from aiogram.types import ChatActions, Message @@ -81,7 +82,8 @@ async def decorator(*args, **kwargs): return result except RetryAfter as error: logger.opt(depth=1).warning( - "Hit flood limit, retry after {} seconds", error.timeout + 1 + "Hit flood limit, retry after {} seconds", + error.timeout + 1, ) await asyncio.sleep(error.timeout + 1) @@ -89,7 +91,7 @@ async def decorator(*args, **kwargs): class Cache: - cached_functions = [] + cached_functions: ClassVar[List[Callable]] = [] @staticmethod def lru(*args, **kwargs): diff --git a/nazurin/utils/exceptions.py b/nazurin/utils/exceptions.py index 8dbc791a..cdcfe4a2 100644 --- a/nazurin/utils/exceptions.py +++ b/nazurin/utils/exceptions.py @@ -9,7 +9,7 @@ def __str__(self): return self.msg -class InvalidCommandUsage(NazurinError): +class InvalidCommandUsageError(NazurinError): """Raised when a command is used incorrectly.""" command: str diff --git a/nazurin/utils/helpers.py b/nazurin/utils/helpers.py index 2d83c662..6f4fad5e 100644 --- a/nazurin/utils/helpers.py +++ b/nazurin/utils/helpers.py @@ -27,6 +27,9 @@ from . import logger +FILENAME_MAX_LENGTH = 255 +CAPTION_MAX_LENGTH = 1024 + async def handle_bad_request(message: Message, error: BadRequest): logger.error("BadRequest exception: {}", error) @@ -37,14 +40,14 @@ async def handle_bad_request(message: Message, error: BadRequest): "Failed to send image as photo, maybe the size is too big, " "consider using download option or try again.\n" f"Message: {message.text}\n" - f"Error: {error}" + f"Error: {error}", ) elif "Group send failed" in str(error): await message.reply( "Failed to send images because one of them is too large, " "consider using download option or try again.\n" f"Message: {message.text}\n" - f"Error: {error}" + f"Error: {error}", ) else: raise error @@ -69,13 +72,14 @@ def sanitize_filename(name: str) -> str: if Path(filename).is_reserved(): filename = "_" + filename name = filename + ext - if len(name) > 255: - name = filename[: 255 - len(ext)] + ext + if len(name) > FILENAME_MAX_LENGTH: + name = filename[: FILENAME_MAX_LENGTH - len(ext)] + ext return name def sanitize_path( - path: os.PathLike, sanitize: Callable[[str], str] = sanitize_path_segment + path: os.PathLike, + sanitize: Callable[[str], str] = sanitize_path_segment, ) -> pathlib.PurePath: """ Remove invalid characters from a path. @@ -91,7 +95,7 @@ def sanitize_path( def sanitize_caption(caption: Caption) -> str: content = caption.text - if len(content) > 1024: + if len(content) > CAPTION_MAX_LENGTH: content = content[:1024] content = escape(content, quote=False) return content @@ -206,8 +210,9 @@ def check_image(path: Union[str, os.PathLike]) -> bool: with Image.open(path) as image: image.verify() + # verify() does not detect all the possible image defects + # e.g. truncated images, try to open the image to detect with Image.open(path) as image: - image = Image.open(path) image.load() return True except OSError as error: @@ -217,8 +222,6 @@ def check_image(path: Union[str, os.PathLike]) -> bool: async def run_in_pool(tasks: Iterable[Coroutine], pool_size: int): scheduler = await aiojobs.create_scheduler(limit=pool_size) - jobs: List[aiojobs.Job] = [] - for task in tasks: - jobs.append(await scheduler.spawn(task)) + jobs: List[aiojobs.Job] = [await scheduler.spawn(task) for task in tasks] await asyncio.gather(*[job.wait() for job in jobs]) await scheduler.close() diff --git a/nazurin/utils/logging.py b/nazurin/utils/logging.py index b340274e..d15ff993 100644 --- a/nazurin/utils/logging.py +++ b/nazurin/utils/logging.py @@ -21,7 +21,8 @@ def emit(self, record): depth += 1 logger.opt(depth=depth, exception=record.exc_info).log( - level, record.getMessage() + level, + record.getMessage(), ) diff --git a/nazurin/utils/network.py b/nazurin/utils/network.py index 6e7d8d1e..d3ff34f2 100644 --- a/nazurin/utils/network.py +++ b/nazurin/utils/network.py @@ -95,7 +95,10 @@ def __init__( @asynccontextmanager async def get( - self, *args, impersonate: str = "chrome110", **kwargs + self, + *args, + impersonate: str = "chrome110", + **kwargs, ) -> AsyncGenerator[CurlResponse, None]: yield await super().request( "GET", @@ -112,7 +115,8 @@ async def download(self, url: str, destination: Union[str, os.PathLike]): async with self.get(url, stream=True) as response: if not response.ok: logger.error( - "Download failed with status code {}", response.status_code + "Download failed with status code {}", + response.status_code, ) logger.info("Response: {}", await response.acontent()) response.raise_for_status() @@ -144,7 +148,9 @@ def __init__( @asynccontextmanager async def get( - self, *args, **kwargs + self, + *args, + **kwargs, ) -> AsyncGenerator[cloudscraper.requests.Response, None]: yield await async_wrap(self.scraper.get)(*args, timeout=self.timeout, **kwargs) @@ -152,7 +158,8 @@ async def download(self, url: str, destination: Union[str, os.PathLike]): async with self.get(url, stream=True) as response: if not response.ok: logger.error( - "Download failed with status code {}", response.status_code + "Download failed with status code {}", + response.status_code, ) logger.info("Response: {}", await response.text) response.raise_for_status() diff --git a/pyproject.toml b/pyproject.toml index 9483b041..6328d24f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,24 @@ -[tool.isort] -profile = "black" \ No newline at end of file +[tool.ruff] +target-version = "py38" + +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "COM", # flake8-commas + "DTZ", # flake8-datetimez + "FBT", # flake8-boolean-trap + "G", # flake8-logging-format + "I", # isort + "INP", # flake8-no-pep420 + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + "PERF", # perflint + "PL", # pylint + "RSE102", # unnecessary-paren-on-raise-exception + "RUF", # ruff + "SIM", # flake8-simplify + "T20", # flake8-print + "UP", # pyupgrade +] +ignore = ["UP007", "PERF203"] \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/database/__init__.py b/tools/database/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/database/helper.py b/tools/database/helper.py index 16f56a22..b42c9a9c 100644 --- a/tools/database/helper.py +++ b/tools/database/helper.py @@ -1,4 +1,3 @@ -# pylint: disable=import-error, wrong-import-position, global-statement """ Helper script to scan artworks from local directory and write metadata to database. Run this script in the root directory of the project. @@ -18,7 +17,8 @@ from nazurin.sites.moebooru.config import COLLECTIONS as MOEBOORU_COLLECTIONS logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.INFO, ) logger = logging.getLogger("helper") sites = SiteManager() @@ -51,7 +51,7 @@ def scan(): "danbooru_new": r"^danbooru (\d+)", "zerochan": r"^Zerochan (\d+)", } - global file_cnt + global file_cnt # noqa: PLW0603 files = glob.glob(directory + "*") for path in files: file_cnt = file_cnt + 1 @@ -59,7 +59,7 @@ def scan(): flag = False source = None - for source, pattern in patterns.items(): + for pattern in patterns.values(): match = re.search(pattern, filename, re.IGNORECASE) if match: flag = True @@ -130,12 +130,12 @@ def print_result(): def main(): - global artworks, success, directory + global artworks, success, directory # noqa: PLW0603 for source in SITES: processed[source] = [] directory = input("Directory path (ends with slash): ") - print("Skip sites(enter site name in SITES, separated with comma):") - print("SITES=" + str(SITES)) + logger.info("Skip sites(enter site name in SITES, separated with comma):") + logger.info("SITES=%s", SITES) skipped = input() skipped = skipped.split(",") db = Database().driver() @@ -143,7 +143,7 @@ def main(): for filename, source, match in scan(): origin_id, site = parse_source(source, match) if source == "danbooru_new": - source = site = "danbooru" + source = site = "danbooru" # noqa: PLW2901 if site in skipped: continue @@ -157,7 +157,6 @@ def main(): try: data = process(filename, source, match, origin_id) - # pylint: disable=broad-except except Exception as err: logger.error("❌ %s: (id: %s) %s", filename, origin_id, err) error.append((filename, origin_id, err)) diff --git a/tools/set_fly_secrets.py b/tools/set_fly_secrets.py index ebdfe0b6..c24d0dac 100644 --- a/tools/set_fly_secrets.py +++ b/tools/set_fly_secrets.py @@ -5,5 +5,5 @@ config = dotenv_values(".env") secrets = " ".join([f'{k}="{v}"' for k, v in config.items()]) command = f"fly secrets set {secrets}" -print(command) +print(command) # noqa: T201 subprocess.run(command, shell=True, check=True)