From 54f2ba7e176806d20c6e4ffa782a32c3378b69c2 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Tue, 17 Dec 2024 16:48:58 +0100 Subject: [PATCH] chore: Remove aiofiles dependency We only write very small files. Unless the artifactsdir is located on a, e.g., very slow network drive, it is very unlikely that the eventloop is blocked by these writes. Usually only a few bytes are written to dumpfiles or to separate result files. Let's use the stdlib instead and get rid of a further dependency. --- pyproject.toml | 2 - src/gallia/command/uds.py | 12 ++--- src/gallia/commands/discover/doip.py | 45 +++++++------------ src/gallia/commands/scan/uds/sa_dump_seeds.py | 8 ++-- src/gallia/dumpcap.py | 2 - src/gallia/utils.py | 5 +-- uv.lock | 22 --------- 7 files changed, 24 insertions(+), 72 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb17a363e..5c94ecdeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ classifiers = [ ] requires-python = ">=3.12,<3.14" dependencies = [ - "aiofiles >=24.1.0,<25.0", "aiosqlite >=0.18", "argcomplete >=2,<4", "boltons>=24.1.0", @@ -65,7 +64,6 @@ dev = [ "reuse >=4.0,<5.0", "ruff >=0.8.0", "sphinx-rtd-theme >=3", - "types-aiofiles >=23.1,<25.0", "types-tabulate >=0.9,<0.10", ] diff --git a/src/gallia/command/uds.py b/src/gallia/command/uds.py index 0a7f881ee..4d507a6f1 100644 --- a/src/gallia/command/uds.py +++ b/src/gallia/command/uds.py @@ -5,7 +5,6 @@ import json from abc import ABC -import aiofiles from pydantic import field_validator from gallia.command.base import FileNames, Scanner, ScannerConfig @@ -138,9 +137,7 @@ async def setup(self) -> None: if self.config.properties is True: path = self.artifacts_dir.joinpath(FileNames.PROPERTIES_PRE.value) - async with aiofiles.open(path, "w") as file: - await file.write(json.dumps(await self.ecu.properties(True), indent=4)) - await file.write("\n") + path.write_text(json.dumps(await self.ecu.properties(True), indent=4) + "\n") if self.db_handler is not None: self._apply_implicit_logging_setting() @@ -156,13 +153,10 @@ async def setup(self) -> None: async def teardown(self) -> None: if self.config.properties is True and (not self.ecu.transport.is_closed): path = self.artifacts_dir.joinpath(FileNames.PROPERTIES_POST.value) - async with aiofiles.open(path, "w") as file: - await file.write(json.dumps(await self.ecu.properties(True), indent=4)) - await file.write("\n") + path.write_text(json.dumps(await self.ecu.properties(True), indent=4) + "\n") path_pre = self.artifacts_dir.joinpath(FileNames.PROPERTIES_PRE.value) - async with aiofiles.open(path_pre) as file: - prop_pre = json.loads(await file.read()) + prop_pre = json.loads(path_pre.read_text()) if self.config.compare_properties and await self.ecu.properties(False) != prop_pre: logger.warning("ecu properties differ, please investigate!") diff --git a/src/gallia/commands/discover/doip.py b/src/gallia/commands/discover/doip.py index 6409b2c2f..2a5893a0c 100644 --- a/src/gallia/commands/discover/doip.py +++ b/src/gallia/commands/discover/doip.py @@ -8,8 +8,6 @@ from itertools import product from urllib.parse import parse_qs, urlparse -import aiofiles - from gallia.command import AsyncScript from gallia.command.base import AsyncScriptConfig from gallia.command.config import AutoInt, Field @@ -294,10 +292,11 @@ async def enumerate_routing_activation_requests( f"doip://{tgt_hostname}:{tgt_port}?protocol_version={self.protocol_version}&activation_type={routing_activation_type:#x}&src_addr={source_address:#x}" ) logger.notice(f"[🤯] Holy moly, it actually worked: {targets[-1]}") - async with aiofiles.open( - self.artifacts_dir.joinpath("1_valid_routing_activation_requests.txt"), "a" + + with self.artifacts_dir.joinpath("1_valid_routing_activation_requests.txt").open( + "a" ) as f: - await f.write(f"{targets[-1]}\n") + f.write(f"{targets[-1]}\n") if len(targets) > 0: logger.notice("[⚔️] It's dangerous to test alone, take one of these:") @@ -340,10 +339,8 @@ async def enumerate_target_addresses( # If we reach this, the request was not denied due to unknown TargetAddress or other DoIP errors known_targets.append(current_target) logger.notice(f"[🥈] HEUREKA: target address {target_addr:#x} is valid! ") - async with aiofiles.open( - self.artifacts_dir.joinpath("3_valid_targets.txt"), "a" - ) as f: - await f.write(f"{current_target}\n") + with self.artifacts_dir.joinpath("3_valid_targets.txt").open("a") as f: + f.write(f"{current_target}\n") # Here is where "reader_task" comes into play, which monitors incoming DiagnosticMessage replies @@ -354,28 +351,22 @@ async def enumerate_target_addresses( elif e.nack_code == DiagnosticMessageNegativeAckCodes.TargetUnreachable: logger.info(f"[💤] {target_addr:#x} is (currently?) unreachable") unreachable_targets.append(current_target) - async with aiofiles.open( - self.artifacts_dir.joinpath("5_unreachable_targets.txt"), "a" - ) as f: - await f.write(f"{current_target}\n") + with self.artifacts_dir.joinpath("5_unreachable_targets.txt").open("a") as f: + f.write(f"{current_target}\n") continue else: logger.warning( f"[🤷] {target_addr:#x} is behaving strangely: {e.nack_code.name}" ) - async with aiofiles.open( - self.artifacts_dir.joinpath("7_targets_with_errors.txt"), "a" - ) as f: - await f.write(f"{target_addr:#x}: {e.nack_code.name}\n") + with self.artifacts_dir.joinpath("7_targets_with_errors.txt").open("a") as f: + f.write(f"{target_addr:#x}: {e.nack_code.name}\n") continue except ConnectionError as e: # Whenever this triggers, but sometimes connections are closed not by us logger.warning(f"[🫦] Sexy, but unexpected: {target_addr:#x} triggered {e!r}") - async with aiofiles.open( - self.artifacts_dir.joinpath("7_targets_with_errors.txt"), "a" - ) as f: - await f.write(f"{target_addr:#x}: {e}\n") + with self.artifacts_dir.joinpath("7_targets_with_errors.txt").open("a") as f: + f.write(f"{target_addr:#x}: {e}\n") # Re-establish DoIP connection await conn.close() await asyncio.sleep(tcp_connect_delay) @@ -426,10 +417,8 @@ async def task_read_diagnostic_messages( if current_target not in responsive_targets: responsive_targets.append(current_target) - async with aiofiles.open( - self.artifacts_dir.joinpath("4_responsive_targets.txt"), "a" - ) as f: - await f.write(f"{current_target}\n") + with self.artifacts_dir.joinpath("4_responsive_targets.txt").open("a") as f: + f.write(f"{current_target}\n") if self.db_handler is not None: await self.db_handler.insert_discovery_result(current_target) @@ -521,10 +510,8 @@ async def run_udp_discovery(self) -> list[tuple[str, int]]: for item in found: url = f"doip://{item[0]}:{item[1]}" logger.notice(url) - async with aiofiles.open( - self.artifacts_dir.joinpath("0_valid_hosts.txt"), "a" - ) as f: - await f.write(f"{url}\n") + with self.artifacts_dir.joinpath("0_valid_hosts.txt").open("a") as f: + f.write(f"{url}\n") else: logger.notice( "[👸] Your princess is in another castle: no DoIP endpoints here it seems..." diff --git a/src/gallia/commands/scan/uds/sa_dump_seeds.py b/src/gallia/commands/scan/uds/sa_dump_seeds.py index c3150cc6b..40d8204e0 100644 --- a/src/gallia/commands/scan/uds/sa_dump_seeds.py +++ b/src/gallia/commands/scan/uds/sa_dump_seeds.py @@ -7,8 +7,6 @@ import time from pathlib import Path -import aiofiles - from gallia.command import UDSScanner from gallia.command.config import AutoInt, Field, HexBytes from gallia.command.uds import UDSScannerConfig @@ -106,7 +104,7 @@ async def main(self) -> None: i = -1 seeds_file = Path.joinpath(self.artifacts_dir, "seeds.bin") - file = await aiofiles.open(seeds_file, "wb", buffering=0) + file = seeds_file.open("wb", buffering=0) duration = self.config.duration * 60 start_time = time.time() last_seed = b"" @@ -148,7 +146,7 @@ async def main(self) -> None: logger.info(f"Received seed of length {len(seed)}") - await file.write(seed) + file.write(seed) if last_seed == seed: logger.warning("Received the same seed as before") @@ -192,6 +190,6 @@ async def main(self) -> None: logger.info(f"Sleeping for {self.config.sleep} seconds between seed requests…") await asyncio.sleep(self.config.sleep) - await file.close() + file.close() self.log_size(seeds_file, time.time() - start_time) await self.ecu.leave_session(session, sleep=self.config.power_cycle_sleep) diff --git a/src/gallia/dumpcap.py b/src/gallia/dumpcap.py index 758acf822..2888c4ec7 100644 --- a/src/gallia/dumpcap.py +++ b/src/gallia/dumpcap.py @@ -116,8 +116,6 @@ async def stop(self) -> None: await self.compressor async def _compressor(self) -> None: - # Gzip support in aiofiles is missing. - # https://github.com/Tinche/aiofiles/issues/46 ready = False assert self.proc.stdout with await asyncio.to_thread(gzip.open, self.outfile, "wb") as f: diff --git a/src/gallia/utils.py b/src/gallia/utils.py index 336bee5bc..265c54965 100644 --- a/src/gallia/utils.py +++ b/src/gallia/utils.py @@ -19,7 +19,6 @@ from typing import TYPE_CHECKING, Any, TypeVar from urllib.parse import urlparse -import aiofiles import pydantic from pydantic.networks import IPvAnyAddress @@ -227,9 +226,9 @@ async def write_target_list( :params db_handler: if given, urls are also written to the database as discovery results :return: None """ - async with aiofiles.open(path, "w") as f: + with path.open("w") as f: for target in targets: - await f.write(f"{target}\n") + f.write(f"{target}\n") if db_handler is not None: await db_handler.insert_discovery_result(str(target)) diff --git a/uv.lock b/uv.lock index 0f4198abb..181c4591e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,15 +1,6 @@ version = 1 requires-python = ">=3.12, <3.14" -[[package]] -name = "aiofiles" -version = "24.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/03/a88171e277e8caa88a4c77808c20ebb04ba74cc4681bf1e9416c862de237/aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c", size = 30247 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/45/30bb92d442636f570cb5651bc661f52b610e2eec3f891a5dc3a4c3667db0/aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5", size = 15896 }, -] - [[package]] name = "aiosqlite" version = "0.20.0" @@ -276,7 +267,6 @@ name = "gallia" version = "1.9.0" source = { editable = "." } dependencies = [ - { name = "aiofiles" }, { name = "aiosqlite" }, { name = "argcomplete" }, { name = "boltons" }, @@ -302,13 +292,11 @@ dev = [ { name = "ruff" }, { name = "sphinx" }, { name = "sphinx-rtd-theme" }, - { name = "types-aiofiles" }, { name = "types-tabulate" }, ] [package.metadata] requires-dist = [ - { name = "aiofiles", specifier = ">=24.1.0,<25.0" }, { name = "aiosqlite", specifier = ">=0.18" }, { name = "argcomplete", specifier = ">=2,<4" }, { name = "boltons", specifier = ">=24.1.0" }, @@ -329,7 +317,6 @@ requires-dist = [ { name = "sphinx", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'dev'", specifier = ">=3" }, { name = "tabulate", specifier = ">=0.9" }, - { name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=23.1,<25.0" }, { name = "types-tabulate", marker = "extra == 'dev'", specifier = ">=0.9,<0.10" }, { name = "zstandard", specifier = ">=0.19" }, ] @@ -974,15 +961,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/b6/a447b5e4ec71e13871be01ba81f5dfc9d0af7e473da256ff46bc0e24026f/tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", size = 37955 }, ] -[[package]] -name = "types-aiofiles" -version = "24.1.0.20240626" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/13/e9/013940b017c313c2e15c64017268fdb0c25e0638621fb8a5d9ebe00fb0f4/types-aiofiles-24.1.0.20240626.tar.gz", hash = "sha256:48604663e24bc2d5038eac05ccc33e75799b0779e93e13d6a8f711ddc306ac08", size = 9357 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/ad/c4b3275d21c5be79487c4f6ed7cd13336997746fe099236cb29256a44a90/types_aiofiles-24.1.0.20240626-py3-none-any.whl", hash = "sha256:7939eca4a8b4f9c6491b6e8ef160caee9a21d32e18534a57d5ed90aee47c66b4", size = 9389 }, -] - [[package]] name = "types-tabulate" version = "0.9.0.20241207"