Skip to content

Commit

Permalink
Use abstract unpacker class
Browse files Browse the repository at this point in the history
  • Loading branch information
TLCFEM committed Oct 14, 2024
1 parent e8732ac commit 391a74c
Show file tree
Hide file tree
Showing 10 changed files with 209 additions and 60 deletions.
5 changes: 0 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ dev = [
"pytest-cov",
"pytest-benchmark",
"pytest-asyncio",
"black",
"ruff",
]
numpy = [
Expand All @@ -46,7 +45,3 @@ numpy = [
"Homepage" = "https://github.com/TLCFEM/msglc"
"Bug Reports" = "https://github.com/TLCFEM/msglc/issuess"
"Source" = "https://github.com/TLCFEM/msglc"

[tool.black]
line-length = 120
fast = true
10 changes: 8 additions & 2 deletions src/msglc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ class FileInfo:


def combine(
archive: str | BytesIO, files: FileInfo | list[FileInfo], *, mode: Literal["a", "w"] = "w", validate: bool = True
archive: str | BytesIO,
files: FileInfo | list[FileInfo],
*,
mode: Literal["a", "w"] = "w",
validate: bool = True,
):
"""
This function is used to combine the multiple serialized files into a single archive.
Expand Down Expand Up @@ -98,7 +102,9 @@ def _iter(path: str | BinaryIO):
combiner.write(_iter(file.path), file.name)


def append(archive: str | BytesIO, files: FileInfo | list[FileInfo], *, validate: bool = True):
def append(
archive: str | BytesIO, files: FileInfo | list[FileInfo], *, validate: bool = True
):
"""
This function is used to append the multiple serialized files to an existing single archive.
Expand Down
10 changes: 8 additions & 2 deletions src/msglc/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ def configure(
This function is used to configure the settings. It accepts any number of keyword arguments.
The function updates the values of the configuration parameters if they are provided in the arguments.
"""
if isinstance(small_obj_optimization_threshold, int) and small_obj_optimization_threshold > 0:
if (
isinstance(small_obj_optimization_threshold, int)
and small_obj_optimization_threshold > 0
):
config.small_obj_optimization_threshold = small_obj_optimization_threshold
if config.trivial_size > config.small_obj_optimization_threshold:
config.trivial_size = config.small_obj_optimization_threshold
Expand All @@ -77,7 +80,10 @@ def configure(
if isinstance(fast_loading, bool):
config.fast_loading = fast_loading

if isinstance(fast_loading_threshold, (int, float)) and 0 <= fast_loading_threshold <= 1:
if (
isinstance(fast_loading_threshold, (int, float))
and 0 <= fast_loading_threshold <= 1
):
config.fast_loading_threshold = fast_loading_threshold

if isinstance(trivial_size, int) and trivial_size > 0:
Expand Down
18 changes: 14 additions & 4 deletions src/msglc/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,18 @@
from msglc import dump
from msglc.config import configure
from msglc.reader import LazyDict, LazyList, LazyStats, LazyReader
from msglc.unpacker import Unpacker


def generate_random_json(depth=10, width=4, simple=False):
seed = random.random()

def generate_token():
return "".join(random.choices(string.ascii_letters + string.digits, k=random.randint(5, 10)))
return "".join(
random.choices(
string.ascii_letters + string.digits, k=random.randint(5, 10)
)
)

if depth == 0 or (simple and seed < 0.1):
return random.choice(
Expand All @@ -44,7 +49,10 @@ def generate_token():
)

if seed < 0.7:
return {generate_token(): generate_random_json(depth - 1, width, True) for _ in range(width)}
return {
generate_token(): generate_random_json(depth - 1, width, True)
for _ in range(width)
}

if seed < 0.95 or not simple:
return [generate_random_json(depth - 1, width, True) for _ in range(width)]
Expand Down Expand Up @@ -106,13 +114,15 @@ def generate(*, depth=6, width=11, threshold=23):
p.map(_dump, step)


def compare(mode, size: int = 13, total: int = 5, unpacker=None):
def compare(mode, size: int = 13, total: int = 5, unpacker: Unpacker = None):
accumulator: int = 0

with open("path.txt", "r") as f:
if mode > 0:
counter = LazyStats()
with LazyReader(f"archive_{size}.msg", counter=counter, unpacker=unpacker) as reader:
with LazyReader(
f"archive_{size}.msg", counter=counter, unpacker=unpacker
) as reader:
while p := f.readline():
accumulator += 1
if accumulator == 10**total:
Expand Down
Loading

0 comments on commit 391a74c

Please sign in to comment.