Skip to content

Commit

Permalink
Package configs (#12)
Browse files Browse the repository at this point in the history
* delete setup.py
* move setup.cfg content to pyproject.toml
* format pyproject.toml
* simplify __init__ files
* remove pypi properties from pyproject.toml
* use statistics instead of pandas
* combine optional dependencies to single dev list
equinor-ruaj authored Nov 10, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 838898e commit f84453b
Showing 9 changed files with 67 additions and 99 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ share/python-wheels/
.installed.cfg
*.egg
MANIFEST
_version.py

# PyInstaller
# Usually these files are written by a python script from a template
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# fmu-sumo-uploader
Upload data from FMU to Sumo
A Python library for uploading from FMU to sumo

### Concepts
`SumoConnection`: The SumoConnection object represents the connection to Sumo, and will handle authentication etc when initiated. This object uses the Sumo python wrapper under the hood.
@@ -52,3 +52,6 @@ case.add_files("/globable/path/to/files/*.gri")
case.upload()

```

## Developer setup
Run: `pip install .[dev]` to also install development requirements.
35 changes: 33 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -3,10 +3,41 @@ requires = ["setuptools", "setuptools-scm", "wheel"]
build-backend = "setuptools.build_meta"

[tool.setuptools_scm]
write_to = "src/fmu/sumo/uploader/version.py"
version_file = "src/fmu/sumo/uploader/_version.py"

[tool.isort]
profile = "black"

[tool.black]
line-length = 79
line-length = 79

[project]
name = "fmu-sumo-uploader"
requires-python = ">=3.8"
dynamic = ["version"]
dependencies = [
"sumo-wrapper-python>=1.0.3",
"fmu-dataio",
"openvds; sys_platform != 'darwin'",
]

[project.optional-dependencies]
dev = ["black", "flake8", "pytest"]

[tool.setuptools]
package-dir = { "" = "src" }
include-package-data = true
platforms = ["any"]

[tool.setuptools.packages.find]
where = ["src"]

[bdist_wheel]
universal = 1

[project.scripts]
sumo_upload = "fmu.sumo.uploader.scripts.sumo_upload:main"

[project.entry-points.ert]
fmu_sumo_jobs = "fmu.sumo.hook_implementations.jobs"
sumo_upload = "fmu.sumo.uploader.scripts.sumo_upload"
64 changes: 0 additions & 64 deletions setup.cfg

This file was deleted.

4 changes: 0 additions & 4 deletions setup.py

This file was deleted.

6 changes: 0 additions & 6 deletions src/fmu/sumo/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +0,0 @@
try:
from .version import version

__version__ = version
except ImportError:
__version__ = "0.0.0"
2 changes: 1 addition & 1 deletion src/fmu/sumo/uploader/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Top-level package for fmu.sumo.uploader"""

try:
from .version import version
from ._version import version

__version__ = version
except ImportError:
47 changes: 28 additions & 19 deletions src/fmu/sumo/uploader/_sumocase.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
import warnings
import time
import datetime
import pandas as pd
import statistics
import httpx


@@ -51,7 +51,8 @@ def _get_sumo_parent_id(self):
except httpx.HTTPStatusError as err:
if err.response.status_code != 404:
logger.warning(
f"Unexpected status code: {err.response.status_code}")
f"Unexpected status code: {err.response.status_code}"
)
return

def upload(self, threads=4, max_attempts=1, register_case=False):
@@ -70,7 +71,8 @@ def upload(self, threads=4, max_attempts=1, register_case=False):
if register_case:
self.register()
logger.info(
"Waiting 1 minute for Sumo to create the case container")
"Waiting 1 minute for Sumo to create the case container"
)
time.sleep(20) # Wait for Sumo to create the container
else:
# We catch the situation where case is not registered on Sumo but
@@ -201,12 +203,18 @@ def _get_log_msg(sumo_parent_id, status):
"case_uuid": str(sumo_parent_id),
"filepath": str(status.get("blob_file_path")),
"metadata": {
"status_code": str(status.get("metadata_upload_response_status_code")),
"status_code": str(
status.get("metadata_upload_response_status_code")
),
"response_text": status.get("metadata_upload_response_text"),
},
"blob": {
"status_code": str(status.get("blob_upload_response_status_code")),
"response_text": ((status.get("blob_upload_response_status_text"))),
"status_code": str(
status.get("blob_upload_response_status_code")
),
"response_text": (
(status.get("blob_upload_response_status_text"))
),
},
}
}
@@ -219,24 +227,25 @@ def _calculate_upload_stats(uploads):
Given a list of results from file upload, calculate and return
timing statistics for uploads."""

df = pd.DataFrame().from_dict(uploads)
blob_upload_times = [u["blob_upload_time_elapsed"] for u in uploads]
metadata_upload_times = [
u["metadata_upload_time_elapsed"] for u in uploads
]

def _get_stats(values):
return {
"mean": statistics.mean(values),
"max": max(values),
"min": min(values),
"std": statistics.stdev(values) if len(values) > 1 else 0.0,
}

stats = {
"blob": {
"upload_time": {
"mean": df["blob_upload_time_elapsed"].mean(),
"max": df["blob_upload_time_elapsed"].max(),
"min": df["blob_upload_time_elapsed"].min(),
"std": df["blob_upload_time_elapsed"].std(),
},
"upload_time": _get_stats(blob_upload_times),
},
"metadata": {
"upload_time": {
"mean": df["metadata_upload_time_elapsed"].mean(),
"max": df["metadata_upload_time_elapsed"].max(),
"min": df["metadata_upload_time_elapsed"].min(),
"std": df["metadata_upload_time_elapsed"].std(),
},
"upload_time": _get_stats(metadata_upload_times),
},
}

2 changes: 0 additions & 2 deletions src/fmu/sumo/uploader/scripts/__init__.py

This file was deleted.

0 comments on commit f84453b

Please sign in to comment.