refactor: major restructure (#5)

kgpayne · Sep 19, 2023 · 339a843 · 339a843
1 parent d5c561e
commit 339a843
Show file tree

Hide file tree

Showing 61 changed files with 945 additions and 773 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -31,6 +31,31 @@ jobs:
       run: |
         poetry run tox -e lint
 
+  mypy:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Only lint using the primary version used for dev
+        python-version: ["3.11"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry==1.2.*
+    - name: Install dependencies
+      run: |
+        poetry install --with dev
+    - name: Run type command from tox.ini
+      run: |
+        poetry run tox -e type
+
   pytest:
 
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -34,6 +34,7 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+!src/singerlake/manifest
 
 # Installer logs
 pip-log.txt

diff --git a/README.md b/README.md
@@ -2,10 +2,12 @@
 
 A library for interacting with a Singerlake; a store of files in Singer JSONL format.
 
+**Note:** This software is in active development. Here be dragons 🐉. Use with caution.
+
 ## Overview
 
 For expended details of the Singerlake Spec, checkout [this blog post](https://kenpayne.co.uk/blog/2023-01-17.html).
 
 Broadly the Singerlake Spec describes a way to write raw Singer messages into object stores (such as S3) in a structured way.
 In addition to offering cheap long-term storage of captured data, it also unlocks several interesting patterns.
-The overall aim would be to roll `python-singerlake` into a `tap-singerlake` and `target-singerlake` for use in data pipelines.
+The overall aim is to roll `python-singerlake` into a `tap-singerlake` and `target-singerlake` for use in data pipelines.
diff --git a/mypy.ini b/mypy.ini
@@ -0,0 +1,2 @@
+[mypy]
+ignore_missing_imports = True
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ version = "0.1.0"
 description = ""
 authors = ["Ken Payne <[email protected]>"]
 readme = "README.md"
-packages = [{include = "singerlake"}]
+packages = [{ include = "singerlake", from = "src" }]
 
 [tool.poetry.dependencies]
 python = ">=3.8,<3.12"
@@ -13,6 +13,7 @@ filelock = "^3.11.0"
 pyfarmhash = "^0.3.2"
 numpy = "^1.24.2"
 base58 = "^2.1.1"
+petname = "^2.6"
 
 [tool.poetry.group.dev.dependencies]
 tox = "^4.4.12"

diff --git a/singerlake/__init__.py b/singerlake/__init__.py
diff --git a/singerlake/models.py b/singerlake/models.py
diff --git a/singerlake/singerlake.py b/singerlake/singerlake.py
diff --git a/singerlake/store/__init__.py b/singerlake/store/__init__.py
diff --git a/singerlake/store/base.py b/singerlake/store/base.py