diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1a30dd7..381fbe3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,31 @@ jobs: run: | poetry run tox -e lint + mypy: + + runs-on: ubuntu-latest + strategy: + matrix: + # Only lint using the primary version used for dev + python-version: ["3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Poetry + run: | + python -m pip install --upgrade pip + pip install poetry==1.2.* + - name: Install dependencies + run: | + poetry install --with dev + - name: Run type command from tox.ini + run: | + poetry run tox -e type + pytest: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 46e60af..88f44b3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ MANIFEST # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec +!src/singerlake/manifest # Installer logs pip-log.txt diff --git a/README.md b/README.md index 6da9d20..4dac702 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,12 @@ A library for interacting with a Singerlake; a store of files in Singer JSONL format. +**Note:** This software is in active development. Here be dragons 🐉. Use with caution. + ## Overview For expended details of the Singerlake Spec, checkout [this blog post](https://kenpayne.co.uk/blog/2023-01-17.html). Broadly the Singerlake Spec describes a way to write raw Singer messages into object stores (such as S3) in a structured way. In addition to offering cheap long-term storage of captured data, it also unlocks several interesting patterns. -The overall aim would be to roll `python-singerlake` into a `tap-singerlake` and `target-singerlake` for use in data pipelines. +The overall aim is to roll `python-singerlake` into a `tap-singerlake` and `target-singerlake` for use in data pipelines. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..1215375 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 58ffbac..701f890 100644 --- a/poetry.lock +++ b/poetry.lock @@ -17,37 +17,34 @@ tests = ["PyHamcrest (>=2.0.2)", "mypy", "pytest (>=4.6)", "pytest-benchmark", " [[package]] name = "black" -version = "23.3.0" +version = "23.9.1" description = "The uncompromising code formatter." category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, - {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, - {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, - {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, - {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, - {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, - {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, - {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, - {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, - {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, - {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, - {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, - {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, - {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, - {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, - {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, - {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, - {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, - {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"}, + {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"}, + {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"}, + {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"}, + {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"}, + {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"}, + {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"}, + {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"}, + {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"}, + {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"}, + {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"}, ] [package.dependencies] @@ -57,7 +54,7 @@ packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -67,38 +64,38 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "cachetools" -version = "5.3.0" +version = "5.3.1" description = "Extensible memoizing collections and decorators" category = "dev" optional = false -python-versions = "~=3.7" +python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, - {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, ] [[package]] name = "chardet" -version = "5.1.0" +version = "5.2.0" description = "Universal encoding detector for Python 3" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, - {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, ] [[package]] name = "click" -version = "8.1.3" +version = "8.1.7" description = "Composable command line interface toolkit" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -118,26 +115,26 @@ files = [ [[package]] name = "distlib" -version = "0.3.6" +version = "0.3.7" description = "Distribution utilities" category = "dev" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, + {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, + {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, ] [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.1.3" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, ] [package.extras] @@ -145,19 +142,20 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.11.0" +version = "3.12.4" description = "A platform independent file lock." category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"}, - {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"}, + {file = "filelock-3.12.4-py3-none-any.whl", hash = "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4"}, + {file = "filelock-3.12.4.tar.gz", hash = "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"}, ] [package.extras] -docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] +typing = ["typing-extensions (>=4.7.1)"] [[package]] name = "iniconfig" @@ -173,49 +171,49 @@ files = [ [[package]] name = "mypy" -version = "1.2.0" +version = "1.5.1" description = "Optional static typing for Python" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"}, - {file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"}, - {file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"}, - {file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"}, - {file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"}, - {file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"}, - {file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"}, - {file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"}, - {file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"}, - {file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"}, - {file = "mypy-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140"}, - {file = "mypy-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336"}, - {file = "mypy-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e"}, - {file = "mypy-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119"}, - {file = "mypy-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a"}, - {file = "mypy-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950"}, - {file = "mypy-1.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6"}, - {file = "mypy-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5"}, - {file = "mypy-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8"}, - {file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"}, - {file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"}, - {file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"}, - {file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"}, - {file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"}, - {file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"}, - {file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"}, + {file = "mypy-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f33592ddf9655a4894aef22d134de7393e95fcbdc2d15c1ab65828eee5c66c70"}, + {file = "mypy-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:258b22210a4a258ccd077426c7a181d789d1121aca6db73a83f79372f5569ae0"}, + {file = "mypy-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9ec1f695f0c25986e6f7f8778e5ce61659063268836a38c951200c57479cc12"}, + {file = "mypy-1.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:abed92d9c8f08643c7d831300b739562b0a6c9fcb028d211134fc9ab20ccad5d"}, + {file = "mypy-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a156e6390944c265eb56afa67c74c0636f10283429171018446b732f1a05af25"}, + {file = "mypy-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ac9c21bfe7bc9f7f1b6fae441746e6a106e48fc9de530dea29e8cd37a2c0cc4"}, + {file = "mypy-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51cb1323064b1099e177098cb939eab2da42fea5d818d40113957ec954fc85f4"}, + {file = "mypy-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596fae69f2bfcb7305808c75c00f81fe2829b6236eadda536f00610ac5ec2243"}, + {file = "mypy-1.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:32cb59609b0534f0bd67faebb6e022fe534bdb0e2ecab4290d683d248be1b275"}, + {file = "mypy-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:159aa9acb16086b79bbb0016145034a1a05360626046a929f84579ce1666b315"}, + {file = "mypy-1.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f6b0e77db9ff4fda74de7df13f30016a0a663928d669c9f2c057048ba44f09bb"}, + {file = "mypy-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26f71b535dfc158a71264e6dc805a9f8d2e60b67215ca0bfa26e2e1aa4d4d373"}, + {file = "mypy-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc3a600f749b1008cc75e02b6fb3d4db8dbcca2d733030fe7a3b3502902f161"}, + {file = "mypy-1.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:26fb32e4d4afa205b24bf645eddfbb36a1e17e995c5c99d6d00edb24b693406a"}, + {file = "mypy-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:82cb6193de9bbb3844bab4c7cf80e6227d5225cc7625b068a06d005d861ad5f1"}, + {file = "mypy-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4a465ea2ca12804d5b34bb056be3a29dc47aea5973b892d0417c6a10a40b2d65"}, + {file = "mypy-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9fece120dbb041771a63eb95e4896791386fe287fefb2837258925b8326d6160"}, + {file = "mypy-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d28ddc3e3dfeab553e743e532fb95b4e6afad51d4706dd22f28e1e5e664828d2"}, + {file = "mypy-1.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:57b10c56016adce71fba6bc6e9fd45d8083f74361f629390c556738565af8eeb"}, + {file = "mypy-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:ff0cedc84184115202475bbb46dd99f8dcb87fe24d5d0ddfc0fe6b8575c88d2f"}, + {file = "mypy-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8f772942d372c8cbac575be99f9cc9d9fb3bd95c8bc2de6c01411e2c84ebca8a"}, + {file = "mypy-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5d627124700b92b6bbaa99f27cbe615c8ea7b3402960f6372ea7d65faf376c14"}, + {file = "mypy-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:361da43c4f5a96173220eb53340ace68cda81845cd88218f8862dfb0adc8cddb"}, + {file = "mypy-1.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:330857f9507c24de5c5724235e66858f8364a0693894342485e543f5b07c8693"}, + {file = "mypy-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:c543214ffdd422623e9fedd0869166c2f16affe4ba37463975043ef7d2ea8770"}, + {file = "mypy-1.5.1-py3-none-any.whl", hash = "sha256:f757063a83970d67c444f6e01d9550a7402322af3557ce7630d3c957386fa8f5"}, + {file = "mypy-1.5.1.tar.gz", hash = "sha256:b031b9601f1060bf1281feab89697324726ba0c0bae9d7cd7ab4b690940f0b92"}, ] [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=3.10" +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] install-types = ["pip"] -python2 = ["typed-ast (>=1.4.0,<2)"] reports = ["lxml"] [[package]] @@ -232,40 +230,40 @@ files = [ [[package]] name = "numpy" -version = "1.24.2" +version = "1.24.4" description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] [[package]] @@ -282,42 +280,53 @@ files = [ [[package]] name = "pathspec" -version = "0.11.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, +] + +[[package]] +name = "petname" +version = "2.6" +description = "Generate human-readable, random object names" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "petname-2.6.tar.gz", hash = "sha256:981c31ef772356a373640d1bb7c67c102e0159eda14578c67a1c99d5b34c9e4c"}, ] [[package]] name = "platformdirs" -version = "3.2.0" +version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, - {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -326,48 +335,48 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pydantic" -version = "1.10.7" +version = "1.10.12" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d"}, - {file = "pydantic-1.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e"}, - {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a"}, - {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f"}, - {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209"}, - {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"}, - {file = "pydantic-1.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a"}, - {file = "pydantic-1.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1"}, - {file = "pydantic-1.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe"}, - {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd"}, - {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb"}, - {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b"}, - {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca"}, - {file = "pydantic-1.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d"}, - {file = "pydantic-1.10.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918"}, - {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe"}, - {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee"}, - {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1"}, - {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a"}, - {file = "pydantic-1.10.7-cp37-cp37m-win_amd64.whl", hash = "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914"}, - {file = "pydantic-1.10.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd"}, - {file = "pydantic-1.10.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245"}, - {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d"}, - {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3"}, - {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52"}, - {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209"}, - {file = "pydantic-1.10.7-cp38-cp38-win_amd64.whl", hash = "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e"}, - {file = "pydantic-1.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143"}, - {file = "pydantic-1.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e"}, - {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d"}, - {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f"}, - {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd"}, - {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5"}, - {file = "pydantic-1.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e"}, - {file = "pydantic-1.10.7-py3-none-any.whl", hash = "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6"}, - {file = "pydantic-1.10.7.tar.gz", hash = "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, + {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, + {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, + {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, + {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, + {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, + {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, + {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, + {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, ] [package.dependencies] @@ -394,34 +403,34 @@ files = [ [[package]] name = "pyproject-api" -version = "1.5.1" +version = "1.6.1" description = "API to interact with the python pyproject.toml based projects" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyproject_api-1.5.1-py3-none-any.whl", hash = "sha256:4698a3777c2e0f6b624f8a4599131e2a25376d90fe8d146d7ac74c67c6f97c43"}, - {file = "pyproject_api-1.5.1.tar.gz", hash = "sha256:435f46547a9ff22cf4208ee274fca3e2869aeb062a4834adfc99a4dd64af3cf9"}, + {file = "pyproject_api-1.6.1-py3-none-any.whl", hash = "sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675"}, + {file = "pyproject_api-1.6.1.tar.gz", hash = "sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538"}, ] [package.dependencies] -packaging = ">=23" +packaging = ">=23.1" tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -testing = ["covdefaults (>=2.2.2)", "importlib-metadata (>=6)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "virtualenv (>=20.17.1)", "wheel (>=0.38.4)"] +docs = ["furo (>=2023.8.19)", "sphinx (<7.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "setuptools (>=68.1.2)", "wheel (>=0.41.2)"] [[package]] name = "pytest" -version = "7.3.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, - {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -433,7 +442,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "tomli" @@ -449,66 +458,66 @@ files = [ [[package]] name = "tox" -version = "4.4.12" +version = "4.11.3" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tox-4.4.12-py3-none-any.whl", hash = "sha256:d4be558809d86fad13f4553976b0500352630a8fbfa39ea4b1ce3bd945ba680b"}, - {file = "tox-4.4.12.tar.gz", hash = "sha256:740f5209d0dec19451b951ee5b1cce4a207acdc7357af84dbc8ec35bcf2c454e"}, + {file = "tox-4.11.3-py3-none-any.whl", hash = "sha256:599af5e5bb0cad0148ac1558a0b66f8fff219ef88363483b8d92a81e4246f28f"}, + {file = "tox-4.11.3.tar.gz", hash = "sha256:5039f68276461fae6a9452a3b2c7295798f00a0e92edcd9a3b78ba1a73577951"}, ] [package.dependencies] -cachetools = ">=5.3" -chardet = ">=5.1" +cachetools = ">=5.3.1" +chardet = ">=5.2" colorama = ">=0.4.6" -filelock = ">=3.11" -packaging = ">=23" -platformdirs = ">=3.2" -pluggy = ">=1" -pyproject-api = ">=1.5.1" +filelock = ">=3.12.3" +packaging = ">=23.1" +platformdirs = ">=3.10" +pluggy = ">=1.3" +pyproject-api = ">=1.6.1" tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} -virtualenv = ">=20.21" +virtualenv = ">=20.24.3" [package.extras] -docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-argparse-cli (>=1.11)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)", "sphinx-copybutton (>=0.5.1)", "sphinx-inline-tabs (>=2022.1.2b11)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] -testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "devpi-process (>=0.3)", "diff-cover (>=7.5)", "distlib (>=0.3.6)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.14)", "psutil (>=5.9.4)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-xdist (>=3.2.1)", "re-assert (>=1.1)", "time-machine (>=2.9)", "wheel (>=0.40)"] +docs = ["furo (>=2023.8.19)", "sphinx (>=7.2.4)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.24)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.1.1)", "devpi-process (>=1)", "diff-cover (>=7.7)", "distlib (>=0.3.7)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.18)", "psutil (>=5.9.5)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-xdist (>=3.3.1)", "re-assert (>=1.1)", "time-machine (>=2.12)", "wheel (>=0.41.2)"] [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] name = "virtualenv" -version = "20.21.0" +version = "20.24.5" description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.21.0-py3-none-any.whl", hash = "sha256:31712f8f2a17bd06234fa97fdf19609e789dd4e3e4bf108c3da71d710651adbc"}, - {file = "virtualenv-20.21.0.tar.gz", hash = "sha256:f50e3e60f990a0757c9b68333c9fdaa72d7188caa417f96af9e52407831a3b68"}, + {file = "virtualenv-20.24.5-py3-none-any.whl", hash = "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b"}, + {file = "virtualenv-20.24.5.tar.gz", hash = "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"}, ] [package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<4" +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<4" [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] -test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "eca2e0b9855012e2b572979bdacc928b97e61ce16eb2e90431c912841b771c5b" +content-hash = "49775160f0532737c14ecf229fb3de71fe16d45d831f01b2c46f4da7cba53d10" diff --git a/pyproject.toml b/pyproject.toml index 77545cb..1161c55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "" authors = ["Ken Payne "] readme = "README.md" -packages = [{include = "singerlake"}] +packages = [{ include = "singerlake", from = "src" }] [tool.poetry.dependencies] python = ">=3.8,<3.12" @@ -13,6 +13,7 @@ filelock = "^3.11.0" pyfarmhash = "^0.3.2" numpy = "^1.24.2" base58 = "^2.1.1" +petname = "^2.6" [tool.poetry.group.dev.dependencies] tox = "^4.4.12" diff --git a/singerlake/__init__.py b/singerlake/__init__.py deleted file mode 100644 index cec464d..0000000 --- a/singerlake/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from singerlake.singerlake import SingerLake - -__all__ = [ - "SingerLake", -] diff --git a/singerlake/models.py b/singerlake/models.py deleted file mode 100644 index f4c3ae8..0000000 --- a/singerlake/models.py +++ /dev/null @@ -1,28 +0,0 @@ -import datetime -from typing import List, Mapping - -from pydantic import BaseModel - - -class LakeManifest(BaseModel): - """Lake Manifest.""" - - -class TapManifest(BaseModel): - """Tap Manifest.""" - - -class StreamManifest(BaseModel): - """Stream Manifest.""" - - files: List[str] = [] - versions: Mapping[str, str] = {} - - def add_files(self, file_names: List[str], schema_hash: str): - """Add files to this Stream Manifest.""" - self.files.extend([f"{schema_hash}/{file_name}" for file_name in file_names]) - if schema_hash not in self.versions: - self.versions[schema_hash] = datetime.datetime.utcnow().strftime( - "%Y%m%dT%H%M%SZ" - ) - return self diff --git a/singerlake/singerlake.py b/singerlake/singerlake.py deleted file mode 100644 index f716021..0000000 --- a/singerlake/singerlake.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, List, Mapping - -import base58 -import farmhash -import numpy as np - -from singerlake.store import BaseStore - - -class SingerLake: - """Singer Lake.""" - - def __init__(self, store: BaseStore): - self.store = store - - def hash_stream_schema(self, stream_schema: Mapping[str, Any]) -> str: - """Calculate a unique short-hash for given schema.""" - data = json.dumps(stream_schema, sort_keys=True) - int64_hash_bytes = ( - np.uint64(farmhash.fingerprint64(data)).astype("int64").tobytes() - ) - return base58.b58encode(int64_hash_bytes).decode("utf-8") - - def write_files( - self, - tap_id: str, - stream_id: str, - stream_schema: Mapping[str, Any], - files: List[Path], - ) -> None: - """Write files to the Lake.""" - stream_schema_hash = self.hash_stream_schema(stream_schema=stream_schema) - self.store.add_files_to_stream( - tap_id=tap_id, - stream_id=stream_id, - stream_schema_hash=stream_schema_hash, - files=files, - ) diff --git a/singerlake/store/__init__.py b/singerlake/store/__init__.py deleted file mode 100644 index 948eb2c..0000000 --- a/singerlake/store/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .base import BaseStore -from .local import LocalStore - -__all__ = [ - "BaseStore", - "LocalStore", -] diff --git a/singerlake/store/base.py b/singerlake/store/base.py deleted file mode 100644 index ddcb683..0000000 --- a/singerlake/store/base.py +++ /dev/null @@ -1,183 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from contextlib import contextmanager -from pathlib import Path -from typing import List, final - -from singerlake.models import BaseModel, LakeManifest, StreamManifest, TapManifest -from singerlake.store.const import ( - LAKE_MANIFEST_FILENAME, - STREAM_MANIFEST_FILENAME, - TAP_MANIFEST_FILENAME, -) - - -class BaseStore(ABC): - """Base SingerLake storage interface.""" - - def __init__(self, lake_root: str): - self._lake_root = lake_root - - @property - def lake_root(self) -> str: - """Lake root dir.""" - return self._lake_root - - @property - def base_prefix(self) -> str: - """Base prefix for Tap paths.""" - return "raw/" - - def get_lake_path(self) -> str: - """Compose root path for this Lake.""" - return self.lake_root + self.base_prefix - - def get_tap_path(self, tap_id: str) -> str: - """Compose path for a given tap_id.""" - return self.get_lake_path() + tap_id - - def get_stream_path( - self, tap_id: str, stream_id: str, stream_schema_hash: str | None = None - ) -> str: - """Compose path for a given Stream by tap_id and stream_id.""" - base_stream_path = self.get_tap_path(tap_id=tap_id) + stream_id - if stream_schema_hash: - return base_stream_path + stream_schema_hash - return base_stream_path - - def get_lake_manifest_path(self) -> str: - """Compose Lake manifest path.""" - return f"{self.get_lake_path()}/{LAKE_MANIFEST_FILENAME}" - - def get_tap_manifest_path(self, tap_id: str) -> str: - """Compose Tap manifest path.""" - return f"{self.get_tap_path(tap_id=tap_id)}/{TAP_MANIFEST_FILENAME}" - - def get_stream_manifest_path(self, tap_id: str, stream_id: str) -> str: - """Compose Stream manifest path.""" - return f"{self.get_stream_path(tap_id=tap_id, stream_id=stream_id)}/{STREAM_MANIFEST_FILENAME}" - - @abstractmethod - def get_manifest(self, manifest_path: str, model: BaseModel) -> BaseModel: - """Get Manifest from path.""" - - @final - def get_lake_manifest(self) -> LakeManifest: - """Read Lake manifest.""" - return self.get_manifest( - manifest_path=self.get_lake_manifest_path(), model=LakeManifest - ) - - @final - def get_tap_manifest(self, tap_id: str) -> TapManifest: - """Read Tap manifest by tap_id.""" - return self.get_manifest( - manifest_path=self.get_tap_manifest_path(tap_id=tap_id), model=TapManifest - ) - - @final - def get_stream_manifest(self, tap_id: str, stream_id: str) -> StreamManifest: - """Read Stream manifest by tap_id and stream_id.""" - return self.get_manifest( - manifest_path=self.get_stream_manifest_path( - tap_id=tap_id, stream_id=stream_id - ), - model=StreamManifest, - ) - - @abstractmethod - def write_manifest(self, manifest_path: str, manifest: BaseModel): - """Write Manifest to given path.""" - - @final - def write_lake_manifest(self, manifest: LakeManifest) -> None: - """Write Lake manifest.""" - self.write_manifest( - manifest_path=self.get_lake_manifest_path(), manifest=manifest - ) - - @final - def write_tap_manifest(self, tap_id: str, manifest: TapManifest) -> None: - """Write Tap manifest.""" - return self.write_manifest( - manifest_path=self.get_tap_manifest_path(tap_id=tap_id), manifest=manifest - ) - - @final - def write_stream_manifest( - self, tap_id: str, stream_id: str, manifest: StreamManifest - ) -> None: - """Write Stream manifest.""" - return self.write_manifest( - manifest_path=self.get_stream_manifest_path( - tap_id=tap_id, stream_id=stream_id - ), - manifest=manifest, - ) - - @contextmanager - @abstractmethod - def lock_lake(self): - """Context manager to acquire and release lock on this SingerLake. - - Locking is required to update the Lake Manifest. - """ - - @contextmanager - @abstractmethod - def lock_tap(self, tap_id: str): - """Context manager to acquire and release lock on a Tap by tap_id. - - Locking is required to update the Tap Manifest. - """ - - @contextmanager - @abstractmethod - def lock_stream(self, tap_id: str, stream_id: str): - """Context manager to acquire and release lock on a Stream by tap_id and stream_id. - - Locking is required to update the Stream Manifest. - """ - - @abstractmethod - def write_files_to_stream(self, stream_path: str, files: List[Path]): - """Write files into Stream path.""" - - @final - def add_files_to_stream( - self, - tap_id: str, - stream_id: str, - stream_schema_hash: str, - files: List[Path], - ) -> None: - """Write files to Lake.""" - stream_path = self.get_stream_path( - tap_id=tap_id, stream_id=stream_id, stream_schema_hash=stream_schema_hash - ) - self.write_files_to_stream(stream_path=stream_path, files=files) - with self.lock_stream(tap_id=tap_id, stream_id=stream_id): - manifest = self.get_stream_manifest(tap_id=tap_id, stream_id=stream_id) - manifest.add_files( - file_names=[file.name for file in files], schema_hash=stream_schema_hash - ) - self.write_stream_manifest( - tap_id=tap_id, stream_id=stream_id, manifest=manifest - ) - - @final - def add_stream_to_tap(self, tap_id: str, stream_id: str) -> None: - """Update Tap manifest with Stream. - - TODO: implement - """ - raise NotImplementedError - - @final - def add_tap_to_lake(self, tap_id: str) -> None: - """Update Lake manifest with Tap. - - TODO: implement - """ - raise NotImplementedError diff --git a/singerlake/store/local.py b/singerlake/store/local.py deleted file mode 100644 index 8425c33..0000000 --- a/singerlake/store/local.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -import json -import shutil -from contextlib import contextmanager -from pathlib import Path -from typing import List - -from filelock import FileLock -from pydantic import BaseModel - -from singerlake.store import BaseStore -from singerlake.store.const import ( - LAKE_MANIFEST_FILENAME, - STREAM_MANIFEST_FILENAME, - TAP_MANIFEST_FILENAME, -) - - -class LocalStore(BaseStore): - """Local Disk SingerLake Store.""" - - def __init__(self, lake_root: Path, lock_timeout: int = 30): - self._lake_root = lake_root - self.lock_timeout = lock_timeout - - @property - def lake_root(self) -> Path: - """Lake root dir.""" - return self._lake_root - - @property - def base_prefix(self) -> str: - """Base prefix for Tap paths.""" - return "raw" - - def get_lake_path(self) -> Path: - """Compose root path for this Lake.""" - return self.lake_root / self.base_prefix - - def get_tap_path(self, tap_id: str) -> Path: - """Compose path for a given Tap by tap_id.""" - return self.get_lake_path() / tap_id - - def get_stream_path( - self, tap_id: str, stream_id: str, stream_schema_hash: str | None = None - ) -> Path: - """Compose path for a given Stream by tap_id and stream_id.""" - base_stream_path = self.get_tap_path(tap_id=tap_id) / stream_id - if stream_schema_hash: - return base_stream_path / stream_schema_hash - return base_stream_path - - def get_lake_manifest_path(self) -> Path: - """Compose Lake manifest path.""" - return self.get_lake_path() / LAKE_MANIFEST_FILENAME - - def get_tap_manifest_path(self, tap_id: str) -> Path: - """Compose Tap manifest path.""" - return self.get_tap_path(tap_id=tap_id) / TAP_MANIFEST_FILENAME - - def get_stream_manifest_path(self, tap_id: str, stream_id: str) -> Path: - """Compose Stream manifest path.""" - return ( - self.get_stream_path(tap_id=tap_id, stream_id=stream_id) - / STREAM_MANIFEST_FILENAME - ) - - def get_manifest(self, manifest_path: Path, model: BaseModel) -> BaseModel: - """Read manifest to at path to given model.""" - if manifest_path.exists(): - with manifest_path.open() as manifest: - data = json.load(manifest) - return model(**data) - return model() - - def write_manifest(self, manifest_path: Path, manifest: BaseModel) -> None: - """Write manifest to given path.""" - manifest_path.parent.mkdir(parents=True, exist_ok=True) - with manifest_path.open("w") as manifest_file: - manifest_file.write(manifest.json()) - - @contextmanager - def lock(self, lockfile_path: Path): - lock = FileLock(lockfile_path, timeout=self.lock_timeout) - with lock: - yield - - @contextmanager - def lock_lake(self): - lockfile_path = Path(f"{self.get_lake_manifest_path()}.lock") - yield self.lock(lockfile_path=lockfile_path) - - @contextmanager - def lock_tap(self, tap_id: str): - lockfile_path = Path(f"{self.get_tap_manifest_path(tap_id=tap_id)}.lock") - yield self.lock(lockfile_path=lockfile_path) - - @contextmanager - def lock_stream(self, tap_id: str, stream_id: str): - lockfile_path = Path( - f"{self.get_stream_manifest_path(tap_id=tap_id, stream_id=stream_id)}.lock" - ) - yield self.lock(lockfile_path=lockfile_path) - - def write_files_to_stream(self, stream_path: Path, files: List[Path]): - for source in files: - destination = stream_path / source.name - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(source, destination) diff --git a/src/singerlake/__init__.py b/src/singerlake/__init__.py new file mode 100644 index 0000000..e49776d --- /dev/null +++ b/src/singerlake/__init__.py @@ -0,0 +1,5 @@ +from .singerlake import Singerlake + +__all__ = [ + "Singerlake", +] diff --git a/src/singerlake/config.py b/src/singerlake/config.py new file mode 100644 index 0000000..7972f88 --- /dev/null +++ b/src/singerlake/config.py @@ -0,0 +1,37 @@ +import typing as t + +from pydantic import BaseModel + + +class GenericPathModel(BaseModel): + """Generic Path Model.""" + + segments: t.Tuple[str, ...] + relative: bool = False + + +class PathConfig(BaseModel): + """Singer Lake Path Config.""" + + path_type: str = "hive" + lake_root: GenericPathModel + + +class LockConfig(BaseModel): + """Singer Lake Lock Config.""" + + lock_type: str = "local" + + +class StoreConfig(BaseModel): + """Singer Lake Store Config.""" + + store_type: str = "local" + path: PathConfig + lock: LockConfig + + +class SingerlakeConfig(BaseModel): + """Singer Lake Config.""" + + store: StoreConfig diff --git a/src/singerlake/discovery/__init__.py b/src/singerlake/discovery/__init__.py new file mode 100644 index 0000000..042faf4 --- /dev/null +++ b/src/singerlake/discovery/__init__.py @@ -0,0 +1,3 @@ +from .discovery_service import DiscoveryService + +__all__ = ["DiscoveryService"] diff --git a/src/singerlake/discovery/discovery_service.py b/src/singerlake/discovery/discovery_service.py new file mode 100644 index 0000000..8daba38 --- /dev/null +++ b/src/singerlake/discovery/discovery_service.py @@ -0,0 +1,32 @@ +import typing as t + +from singerlake.tap import Tap + +if t.TYPE_CHECKING: + from singerlake import Singerlake + + +class DiscoveryService: + """Discovery Service. + + This service is responsible for discovering Taps available in a given Singerlake. + """ + + def __init__(self, singerlake: "Singerlake"): + self.singerlake = singerlake + self._tap_cache: t.List[str] | None = None + + def list_taps(self) -> t.List[str]: + """List available Taps.""" + if self._tap_cache is None: + lake_manifest = self.singerlake.manifest_service.lake_manifest + self._tap_cache = lake_manifest.taps + + return self._tap_cache + + def get_tap(self, tap_id): + """Get a Tap by ID.""" + tap_manifest = self.singerlake.manifest_service.get_tap_manifest(tap_id=tap_id) + if tap_manifest: + return Tap(singerlake=self.singerlake, tap_manifest=tap_manifest) + raise ValueError(f"Tap {tap_id} not found.") diff --git a/src/singerlake/manifest/__init__.py b/src/singerlake/manifest/__init__.py new file mode 100644 index 0000000..b424a66 --- /dev/null +++ b/src/singerlake/manifest/__init__.py @@ -0,0 +1,4 @@ +from .manifest_service import ManifestService +from .models import LakeManifest, StreamManifest, TapManifest + +__all__ = ["ManifestService", "LakeManifest", "StreamManifest", "TapManifest"] diff --git a/src/singerlake/manifest/base.py b/src/singerlake/manifest/base.py new file mode 100644 index 0000000..e69de29 diff --git a/src/singerlake/manifest/manifest_service.py b/src/singerlake/manifest/manifest_service.py new file mode 100644 index 0000000..4c99e08 --- /dev/null +++ b/src/singerlake/manifest/manifest_service.py @@ -0,0 +1,39 @@ +import typing as t + +from .models import LakeManifest, StreamManifest, TapManifest + +if t.TYPE_CHECKING: + from singerlake import Singerlake + + +class ManifestService: + """Manifest Service.""" + + def __init__(self, singerlake: "Singerlake"): + self.singerlake = singerlake + + self._lake_manifest: LakeManifest | None = None + + @property + def lake_manifest(self): + """Get the Lake Manifest.""" + if self._lake_manifest is None: + raw_lake_manifest = self.singerlake.store.read_lake_manifest() + if raw_lake_manifest: + self._lake_manifest = LakeManifest(**raw_lake_manifest) + + return self._lake_manifest + + def get_tap_manifest(self, tap_id: str): + """Get a Tap Manifest by ID.""" + tap_manifest = self.singerlake.store.read_tap_manifest(tap_id=tap_id) + if tap_manifest: + return TapManifest(**tap_manifest) + + def get_stream_manifest(self, tap_id: str, stream_id: str): + """Get a Stream Manifest by ID.""" + stream_manifest = self.singerlake.store.read_stream_manifest( + tap_id=tap_id, stream_id=stream_id + ) + if stream_manifest: + return StreamManifest(**stream_manifest) diff --git a/src/singerlake/manifest/models.py b/src/singerlake/manifest/models.py new file mode 100644 index 0000000..faf27d8 --- /dev/null +++ b/src/singerlake/manifest/models.py @@ -0,0 +1,27 @@ +import datetime +import typing as t + +from pydantic import BaseModel + + +class StreamManifest(BaseModel): + """Stream Manifest.""" + + stream_id: str + + files: t.List[str] = [] + versions: t.Mapping[str, str] = {} + + +class TapManifest(BaseModel): + """Tap Manifest.""" + + tap_id: str + streams: t.List[str] = [] + + +class LakeManifest(BaseModel): + """Lake Manifest.""" + + lake_id: str + taps: t.List[str] = [] diff --git a/src/singerlake/singerlake.py b/src/singerlake/singerlake.py new file mode 100644 index 0000000..a22426e --- /dev/null +++ b/src/singerlake/singerlake.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import typing as t +from uuid import uuid4 + +from singerlake.config import SingerlakeConfig +from singerlake.discovery import DiscoveryService +from singerlake.manifest import ManifestService +from singerlake.store import StoreService + +if t.TYPE_CHECKING: + from singerlake.tap import Tap + + +class Singerlake: + """Singer Lake.""" + + def __init__(self, config: dict | None = None): + """Initialize a Singer Lake instance.""" + config_dict = config or {} + + self.instance_id = str(uuid4()) + self.config = SingerlakeConfig(**config_dict) + self.manifest_service = ManifestService(singerlake=self) + self.discovery_service = DiscoveryService(singerlake=self) + self.store = StoreService(singerlake=self, config=self.config.store).get_store() + + self._lake_id: str | None = None + + @property + def lake_id(self) -> str: + """Return the Lake ID.""" + if self._lake_id is None: + self._lake_id = self.manifest_service.lake_manifest.lake_id + return self._lake_id + + def list_taps(self) -> list[str]: + """Return Taps stored in this Singerlake.""" + return self.discovery_service.list_taps() + + def get_tap(self, tap_id: str) -> "Tap": + """Return a Tap stored in this Singerlake.""" + return self.discovery_service.get_tap(tap_id=tap_id) diff --git a/src/singerlake/store/__init__.py b/src/singerlake/store/__init__.py new file mode 100644 index 0000000..15e1534 --- /dev/null +++ b/src/singerlake/store/__init__.py @@ -0,0 +1,17 @@ +from .base import BaseStore +from .local import LocalStore +from .path_manager.constant import ( + LAKE_MANIFEST_FILENAME, + STREAM_MANIFEST_FILENAME, + TAP_MANIFEST_FILENAME, +) +from .store_service import StoreService + +__all__ = [ + "StoreService", + "BaseStore", + "LocalStore", + "LAKE_MANIFEST_FILENAME", + "TAP_MANIFEST_FILENAME", + "STREAM_MANIFEST_FILENAME", +] diff --git a/src/singerlake/store/base.py b/src/singerlake/store/base.py new file mode 100644 index 0000000..189b131 --- /dev/null +++ b/src/singerlake/store/base.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import typing as t +from abc import ABC + +if t.TYPE_CHECKING: + from singerlake.store.locker.base import BaseLocker + from singerlake.store.path_manager.base import BasePathManager + + +class BaseStore(ABC): + """Base SingerLake storage interface.""" + + def __init__(self, locker: "BaseLocker", path_manager: "BasePathManager") -> None: + """Base SingerLake storage interface.""" + self.locker = locker + self.path_manager = path_manager + + def read_tap_manifest(self, tap_id: str) -> dict | None: + """Read a Tap Manifest.""" + raise NotImplementedError() + + def read_stream_manifest(self, tap_id: str, stream_id: str) -> dict | None: + """Read a Stream Manifest.""" + raise NotImplementedError() diff --git a/src/singerlake/store/local.py b/src/singerlake/store/local.py new file mode 100644 index 0000000..195dbf0 --- /dev/null +++ b/src/singerlake/store/local.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import hashlib +import json +import typing as t +from pathlib import Path + +from .base import BaseStore + +if t.TYPE_CHECKING: + from singerlake.store.path_manager.base import GenericPath + + from .locker.base import BaseLocker + from .path_manager.base import BasePathManager + + +class LocalStore(BaseStore): + """Local directory store.""" + + def __init__(self, locker: "BaseLocker", path_manager: "BasePathManager"): + """Local directory store.""" + super().__init__(locker=locker, path_manager=path_manager) + + self._lake_manifest_checksum: str | None = None + + def _md5(self, file_path: Path): + """Return the md5 checksum of a file.""" + hash_md5 = hashlib.md5() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + def _read_json(self, file_path: Path): + """Read a JSON file.""" + with file_path.open("r", encoding="utf-8") as json_file: + return json.load(json_file) + + def _to_path(self, generic_path: "GenericPath") -> Path: + """Convert a GenericPath to a pathlib Path.""" + return Path(*generic_path.segments) + + # Lake Manifest + def read_lake_manifest_checksum(self) -> str | None: + """Read the Lake Manifest checksum.""" + lake_manifest_path = self._to_path(self.path_manager.lake_manifest_path) + return self._md5(lake_manifest_path) + + def read_lake_manifest(self) -> tuple[dict, str] | None: + """Read the Lake Manifest.""" + lake_manifest_path = self._to_path(self.path_manager.lake_manifest_path) + lake_manifest = self._read_json(lake_manifest_path) + if lake_manifest is not None: + self._lake_manifest_checksum = self.read_lake_manifest_checksum() + return lake_manifest + return None + + @property + def lake_manifest_has_changed(self) -> bool: + """Return True if the Lake Manifest has changed.""" + return self.read_lake_manifest_checksum() != self._lake_manifest_checksum + + # Tap Manifest + def read_tap_manifest(self, tap_id: str) -> dict | None: + """Read a Tap Manifest.""" + tap_manifest_path = self._to_path( + self.path_manager.get_tap_manifest_path(tap_id=tap_id) + ) + return self._read_json(tap_manifest_path) + + # Stream Manifest + def read_stream_manifest(self, tap_id: str, stream_id: str) -> dict | None: + """Read a Stream Manifest.""" + stream_manifest_path = self._to_path( + self.path_manager.get_stream_manifest_path( + tap_id=tap_id, stream_id=stream_id + ) + ) + return self._read_json(stream_manifest_path) diff --git a/src/singerlake/store/locker/README.md b/src/singerlake/store/locker/README.md new file mode 100644 index 0000000..68b8de5 --- /dev/null +++ b/src/singerlake/store/locker/README.md @@ -0,0 +1,21 @@ +## Example Approach + +Let the service in question who needs the lock be called S1. + +1. The S1 has a UUID as a static in-memory variable - S1-UUID. + +2. First read the file s3://bucket/locks/lock.json with JSON format: {UUID:'1-2-3-4' , createdTime:yyyy-mm-dd-hh-mm-ss-ms}; + +3. Does the UUID from JSON match with S1-UUID? yes? then you already have the lock. return true. If the string doesn't match... + +4. Was the lock created more than abort time? yes? Then move to step 5. No? then return saying you don't have the lock. return false. + +5. Write S1-UUID and the current time to JSON and write it to s3://bucket/lock/lock.json + +6. Wait for 250ms --250 because I have never seen S3 latency to be more than 125ms; so taking double time to play safe. Read the lock.json again. Read the UUID from the JSON if the UUID matches S1-UUID then you get the lock. return true. If not return false. + +For HTTP API calls I put the abort time as 2sec (because HTTP SLA is 2sec to our APIs). For spark jobs with higher SLA, we put 2hrs --because the spark jobs take 2hrs to complete. + +## Example Implementations + +- https://github.com/jfstephe/aws-s3-lock diff --git a/src/singerlake/store/locker/__init__.py b/src/singerlake/store/locker/__init__.py new file mode 100644 index 0000000..0cf5a99 --- /dev/null +++ b/src/singerlake/store/locker/__init__.py @@ -0,0 +1,3 @@ +from .lock_service import LockService + +__all__ = ["LockService"] diff --git a/src/singerlake/store/locker/base.py b/src/singerlake/store/locker/base.py new file mode 100644 index 0000000..9e7d7fb --- /dev/null +++ b/src/singerlake/store/locker/base.py @@ -0,0 +1,72 @@ +import typing as t + +if t.TYPE_CHECKING: + from singerlake import Singerlake + + +class BaseLocker: + """Base Lock.""" + + def __init__(self, singerlake: "Singerlake"): + """Base Lock. + + Args: + store: SingerLake Store. + stream: SingerLake Stream. + """ + self.singerlake = singerlake + + @property + def instance_id(self): + """Instance ID.""" + return self.singerlake.instance_id + + @property + def store(self): + """Store.""" + return self.singerlake.store + + def acquire(self): + """Acquire lock. + + If a stream is provided, acquire a lock on that stream. + Otherwise, acquire a lock on the lake. + """ + raise NotImplementedError() + + def release(self): + """Release lock. + + If a stream is provided, release the lock on that stream. + Otherwise, release the lock on the lake. + """ + raise NotImplementedError() + + def refresh(self): + """Refresh lock. + + If a stream is provided, refresh the lock on that stream. + Otherwise, refresh the lock on the lake. + """ + raise NotImplementedError() + + def __enter__(self): + """Enter context.""" + self.acquire() + + def __exit__(self, exc_type, exc_value, traceback): + """Exit context.""" + self.release() + + +class LocalFileLock(BaseLocker): + """Local File Lock.""" + + def acquire(self): + return super().acquire() + + def release(self): + return super().release() + + def refresh(self): + return super().refresh() diff --git a/src/singerlake/store/locker/local_file.py b/src/singerlake/store/locker/local_file.py new file mode 100644 index 0000000..e69de29 diff --git a/src/singerlake/store/locker/lock_service.py b/src/singerlake/store/locker/lock_service.py new file mode 100644 index 0000000..9e2f49b --- /dev/null +++ b/src/singerlake/store/locker/lock_service.py @@ -0,0 +1,15 @@ +import typing as t + +from .base import BaseLocker + +if t.TYPE_CHECKING: + from singerlake import Singerlake + from singerlake.config import LockConfig + + +class LockService: + def __init__(self, config: "LockConfig"): + self.config = config + + def get_locker(self, singerlake: "Singerlake"): + return BaseLocker(singerlake=singerlake) diff --git a/src/singerlake/store/path_manager/__init__.py b/src/singerlake/store/path_manager/__init__.py new file mode 100644 index 0000000..b9ce6ab --- /dev/null +++ b/src/singerlake/store/path_manager/__init__.py @@ -0,0 +1,15 @@ +from .base import BasePathManager +from .constant import ( + LAKE_MANIFEST_FILENAME, + STREAM_MANIFEST_FILENAME, + TAP_MANIFEST_FILENAME, +) +from .path_service import PathService + +__all__ = [ + "BasePathManager", + "PathService", + "LAKE_MANIFEST_FILENAME", + "TAP_MANIFEST_FILENAME", + "STREAM_MANIFEST_FILENAME", +] diff --git a/src/singerlake/store/path_manager/base.py b/src/singerlake/store/path_manager/base.py new file mode 100644 index 0000000..c4e508c --- /dev/null +++ b/src/singerlake/store/path_manager/base.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import json +import typing as t + +import base58 +import farmhash +import numpy as np + +from .constant import ( + LAKE_MANIFEST_FILENAME, + STREAM_MANIFEST_FILENAME, + TAP_MANIFEST_FILENAME, +) + +if t.TYPE_CHECKING: + from singerlake.config import PathConfig + + +class GenericPath: + + """Generic path class.""" + + def __init__(self, segments: tuple[str, ...], relative: bool = False): + self.segments = segments + self.relative = relative + + def __str__(self): + return "/".join(self.segments) + + def __repr__(self): + return f"GenericPath({self.segments})" + + def __eq__(self, other): + return (self.segments == other.segments) and (self.relative == other.relative) + + def __hash__(self): + return hash(self.segments) + + def extend(self, *args: str) -> "GenericPath": + """Extend the path.""" + return GenericPath(self.segments + args, relative=self.relative) + + @classmethod + def from_dict(cls, data: t.Mapping[str, t.Any]) -> "GenericPath": + """Extend the path with a dict.""" + return GenericPath(data["segments"], relative=data.get("relative", False)) + + @classmethod + def from_model(cls, model: t.Any) -> "GenericPath": + """Extend the path with a dict.""" + return GenericPath(model.segments, relative=model.relative) + + +class BasePathManager: + def __init__(self, config: "PathConfig"): + self.config = config + self.lake_root = GenericPath.from_model(self.config.lake_root) + + def hash_stream_schema(self, stream_schema: t.Mapping[str, t.Any]) -> str: + """Calculate a unique short-hash for given schema.""" + data = json.dumps(stream_schema, sort_keys=True) + int64_hash_bytes = ( + np.uint64(farmhash.fingerprint64(data)).astype("int64").tobytes() + ) + return base58.b58encode(int64_hash_bytes).decode("utf-8") + + @property + def lake_manifest_path(self) -> GenericPath: + """Get the lake manifest path.""" + return self.lake_root.extend(*("raw", LAKE_MANIFEST_FILENAME)) + + def get_tap_manifest_path(self, tap_id: str) -> GenericPath: + """Get the tap manifest path.""" + return self.lake_root.extend(*("raw", tap_id, TAP_MANIFEST_FILENAME)) + + def get_stream_manifest_path(self, tap_id: str, stream_id: str) -> GenericPath: + """Get the stream manifest path.""" + return self.lake_root.extend( + *("raw", tap_id, stream_id, STREAM_MANIFEST_FILENAME) + ) diff --git a/singerlake/store/const.py b/src/singerlake/store/path_manager/constant.py similarity index 100% rename from singerlake/store/const.py rename to src/singerlake/store/path_manager/constant.py diff --git a/src/singerlake/store/path_manager/hive.py b/src/singerlake/store/path_manager/hive.py new file mode 100644 index 0000000..30c950a --- /dev/null +++ b/src/singerlake/store/path_manager/hive.py @@ -0,0 +1,5 @@ +from .base import BasePathManager + + +class HivePathManager(BasePathManager): + """HivePathManager is a path manager for Hive paths.""" diff --git a/src/singerlake/store/path_manager/path_service.py b/src/singerlake/store/path_manager/path_service.py new file mode 100644 index 0000000..4cff1ac --- /dev/null +++ b/src/singerlake/store/path_manager/path_service.py @@ -0,0 +1,20 @@ +from singerlake.config import PathConfig + +from .base import BasePathManager +from .hive import HivePathManager + + +class PathService: + """PathService is a factory class that returns a PathManager based on the + config provided. + """ + + def __init__(self, config: PathConfig): + self.config = config + + def get_path_manager(self) -> BasePathManager: + """Return a path manager instance.""" + if self.config.path_type == "hive": + return HivePathManager(config=self.config) + + raise ValueError(f"Unknown path type: {self.config.path_type}") diff --git a/src/singerlake/store/store_service.py b/src/singerlake/store/store_service.py new file mode 100644 index 0000000..ecc57b9 --- /dev/null +++ b/src/singerlake/store/store_service.py @@ -0,0 +1,28 @@ +import typing as t + +from .local import LocalStore +from .locker import LockService +from .path_manager import PathService + +if t.TYPE_CHECKING: + from singerlake import Singerlake + from singerlake.config import StoreConfig + from singerlake.store.base import BaseStore + + +class StoreService: + def __init__(self, singerlake: "Singerlake", config: "StoreConfig"): + self.singerlake = singerlake + self.config = config + + def get_store(self) -> "BaseStore": + """Return a store instance.""" + locker = LockService(config=self.config.lock).get_locker( + singerlake=self.singerlake + ) + path_manager = PathService(config=self.config.path).get_path_manager() + + if self.config.store_type == "local": + return LocalStore(locker=locker, path_manager=path_manager) + + raise ValueError(f"Unknown store type: {self.config.store_type}") diff --git a/src/singerlake/stream/__init__.py b/src/singerlake/stream/__init__.py new file mode 100644 index 0000000..1201730 --- /dev/null +++ b/src/singerlake/stream/__init__.py @@ -0,0 +1,3 @@ +from .stream import Stream + +__all__ = ["Stream"] diff --git a/src/singerlake/stream/stream.py b/src/singerlake/stream/stream.py new file mode 100644 index 0000000..a0b4a10 --- /dev/null +++ b/src/singerlake/stream/stream.py @@ -0,0 +1,23 @@ +import typing as t + +from singerlake.manifest.models import StreamManifest + +if t.TYPE_CHECKING: + from singerlake import Singerlake + from singerlake.tap import Tap + + +class Stream: + def __init__( + self, + singerlake: "Singerlake", + tap: "Tap", + stream_manifest: StreamManifest, + ) -> None: + self.singerlake = singerlake + self.tap = tap + self.stream_manifest = stream_manifest + + @property + def stream_id(self) -> str: + return self.stream_manifest.stream_id diff --git a/src/singerlake/tap/__init__.py b/src/singerlake/tap/__init__.py new file mode 100644 index 0000000..9457188 --- /dev/null +++ b/src/singerlake/tap/__init__.py @@ -0,0 +1,3 @@ +from .tap import Tap + +__all__ = ["Tap"] diff --git a/src/singerlake/tap/tap.py b/src/singerlake/tap/tap.py new file mode 100644 index 0000000..da88dec --- /dev/null +++ b/src/singerlake/tap/tap.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import typing as t + +from singerlake.stream import Stream + +if t.TYPE_CHECKING: + from singerlake import Singerlake + from singerlake.manifest.models import TapManifest + + +class Tap: + """Tap.""" + + def __init__(self, singerlake: "Singerlake", tap_manifest: "TapManifest") -> None: + """Tap.""" + self.singerlake = singerlake + self.tap_manifest = tap_manifest + self._stream_cache: t.Mapping[str, t.Any] | None = None + + @property + def tap_id(self) -> str: + """Tap ID.""" + return self.tap_manifest.tap_id + + @property + def streams(self) -> t.Mapping[str, Stream]: + """Streams.""" + if self._stream_cache is None: + self._stream_cache = {} + for stream_id in self.tap_manifest.streams: + stream_manifest = self.singerlake.manifest_service.get_stream_manifest( + tap_id=self.tap_id, stream_id=stream_id + ) + self._stream_cache[stream_id] = Stream( + singerlake=self.singerlake, + tap=self, + stream_manifest=stream_manifest, + ) + return self._stream_cache + + @property + def stream_ids(self) -> list[str]: + """Stream IDs.""" + return self.tap_manifest.streams + + def get_stream(self, stream_id: str) -> Stream | None: + """Get Stream.""" + return self.streams.get(stream_id) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..193f3fd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +import pytest + +from singerlake import Singerlake + + +@pytest.fixture(scope="session") +def singerlake_config(): + return { + "store": { + "store_type": "local", + "path": { + "path_type": "hive", + "lake_root": { + "segments": ("tests", "data", "lake"), + "relative": True, + }, + }, + "lock": { + "lock_type": "local", + }, + } + } + + +@pytest.fixture(scope="session") +def singerlake(singerlake_config: dict): + yield Singerlake(config=singerlake_config) diff --git a/tests/data/lake/raw/manifest.json b/tests/data/lake/raw/manifest.json new file mode 100644 index 0000000..d5f50f9 --- /dev/null +++ b/tests/data/lake/raw/manifest.json @@ -0,0 +1,4 @@ +{ + "lake_id": "sound-oryx", + "taps": ["tap-carbon-intensity"] +} diff --git a/tests/data/lake/raw/tap-carbon-intensity/entry/manifest.json b/tests/data/lake/raw/tap-carbon-intensity/entry/manifest.json new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/lake/raw/tap-carbon-intensity/generationmix/manifest.json b/tests/data/lake/raw/tap-carbon-intensity/generationmix/manifest.json new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/lake/raw/tap-carbon-intensity/manifest.json b/tests/data/lake/raw/tap-carbon-intensity/manifest.json new file mode 100644 index 0000000..d7977e9 --- /dev/null +++ b/tests/data/lake/raw/tap-carbon-intensity/manifest.json @@ -0,0 +1,4 @@ +{ + "tap_id": "tap-carbon-intensity", + "streams": ["entry", "generationmix", "region"] +} diff --git a/tests/data/lake/raw/tap-carbon-intensity/region/manifest.json b/tests/data/lake/raw/tap-carbon-intensity/region/manifest.json new file mode 100644 index 0000000..e69de29 diff --git a/tests/example_files/entry-20230414T155444-20230414T155444.singer.gz b/tests/example_files/entry-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index e17487c..0000000 Binary files a/tests/example_files/entry-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/example_files/entry.schema.json b/tests/example_files/entry.schema.json deleted file mode 100644 index 5efca2b..0000000 --- a/tests/example_files/entry.schema.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": "SCHEMA", - "stream": "entry", - "schema": { - "type": "object", - "properties": { - "id": { "type": "string" }, - "from": { "type": "string", "format": "date-time" }, - "to": { "type": "string", "format": "date-time" }, - "forecast": { "type": "integer" }, - "index": { "type": "string" }, - "region_id": { "type": "integer" }, - "_sdc_extracted_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "_sdc_received_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_deleted_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_batched_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_table_version": { "type": ["null", "integer"] }, - "_sdc_sequence": { "type": ["null", "integer"] } - } - }, - "key_properties": ["id"] -} diff --git a/tests/example_files/generationmix-20230414T155444-20230414T155444.singer.gz b/tests/example_files/generationmix-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index 05e9307..0000000 Binary files a/tests/example_files/generationmix-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/example_files/generationmix.schema.json b/tests/example_files/generationmix.schema.json deleted file mode 100644 index f902030..0000000 --- a/tests/example_files/generationmix.schema.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "type": "SCHEMA", - "stream": "generationmix", - "schema": { - "type": "object", - "properties": { - "id": { "type": "string" }, - "fuel": { "type": "string" }, - "perc": { "type": "number" }, - "entry_id": { "type": "string" }, - "_sdc_extracted_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "_sdc_received_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_deleted_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_batched_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_table_version": { "type": ["null", "integer"] }, - "_sdc_sequence": { "type": ["null", "integer"] } - } - }, - "key_properties": ["id"] -} diff --git a/tests/example_files/region-20230414T155444-20230414T155444.singer.gz b/tests/example_files/region-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index 3e67711..0000000 Binary files a/tests/example_files/region-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/example_files/region.schema.json b/tests/example_files/region.schema.json deleted file mode 100644 index 3536aa2..0000000 --- a/tests/example_files/region.schema.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": "SCHEMA", - "stream": "region", - "schema": { - "type": "object", - "properties": { - "id": { "type": "integer" }, - "dnoregion": { "type": "string" }, - "shortname": { "type": "string" }, - "_sdc_extracted_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "_sdc_received_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_deleted_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_batched_at": { "type": ["null", "string"], "format": "date-time" }, - "_sdc_table_version": { "type": ["null", "integer"] }, - "_sdc_sequence": { "type": ["null", "integer"] } - } - }, - "key_properties": ["id"] -} diff --git a/tests/local_singerlake/.gitignore b/tests/local_singerlake/.gitignore deleted file mode 100644 index 1806ddc..0000000 --- a/tests/local_singerlake/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -./* -!.gitignore \ No newline at end of file diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/Y8Mjkb4i9yM/entry-20230414T155444-20230414T155444.singer.gz b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/Y8Mjkb4i9yM/entry-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index e17487c..0000000 Binary files a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/Y8Mjkb4i9yM/entry-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/manifest.json b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/manifest.json deleted file mode 100644 index 5ac7c17..0000000 --- a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/entry/manifest.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "files": ["Y8Mjkb4i9yM/entry-20230414T155444-20230414T155444.singer.gz"], - "versions": { "Y8Mjkb4i9yM": "20230420T161758Z" } -} diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/XcxC17T7smW/generationmix-20230414T155444-20230414T155444.singer.gz b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/XcxC17T7smW/generationmix-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index 05e9307..0000000 Binary files a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/XcxC17T7smW/generationmix-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/manifest.json b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/manifest.json deleted file mode 100644 index b0d0ff4..0000000 --- a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/generationmix/manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"files": ["XcxC17T7smW/generationmix-20230414T155444-20230414T155444.singer.gz"], "versions": {"XcxC17T7smW": "20230420T161758Z"}} \ No newline at end of file diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/3v36Q5QKNZM/region-20230414T155444-20230414T155444.singer.gz b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/3v36Q5QKNZM/region-20230414T155444-20230414T155444.singer.gz deleted file mode 100644 index 3e67711..0000000 Binary files a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/3v36Q5QKNZM/region-20230414T155444-20230414T155444.singer.gz and /dev/null differ diff --git a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/manifest.json b/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/manifest.json deleted file mode 100644 index 6d2e0f2..0000000 --- a/tests/local_singerlake/test_local_singerlake/raw/tap-carbon-intensity/region/manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"files": ["3v36Q5QKNZM/region-20230414T155444-20230414T155444.singer.gz"], "versions": {"3v36Q5QKNZM": "20230420T161758Z"}} \ No newline at end of file diff --git a/tests/test_local_singerlake.py b/tests/test_local_singerlake.py deleted file mode 100644 index 9213ae6..0000000 --- a/tests/test_local_singerlake.py +++ /dev/null @@ -1,85 +0,0 @@ -import json -import shutil -from pathlib import Path - -import pytest - -from singerlake import SingerLake -from singerlake.store import LocalStore - -EXAMPLE_FILES_DIR = Path(__file__).parent / "example_files" -TEST_DIR = Path(__file__).parent / "local_singerlake" / "test_local_singerlake" -LAKE_PATH = TEST_DIR / "raw" -EXAMPLE_TAP_SYNC = { - "tap-carbon-intensity": { - "entry": { - "files": [ - EXAMPLE_FILES_DIR / "entry-20230414T155444-20230414T155444.singer.gz" - ], - "schema": EXAMPLE_FILES_DIR / "entry.schema.json", - "schema_hash": "Y8Mjkb4i9yM", - }, - "generationmix": { - "files": [ - EXAMPLE_FILES_DIR - / "generationmix-20230414T155444-20230414T155444.singer.gz" - ], - "schema": EXAMPLE_FILES_DIR / "generationmix.schema.json", - "schema_hash": "XcxC17T7smW", - }, - "region": { - "files": [ - EXAMPLE_FILES_DIR / "region-20230414T155444-20230414T155444.singer.gz" - ], - "schema": EXAMPLE_FILES_DIR / "region.schema.json", - "schema_hash": "3v36Q5QKNZM", - }, - } -} - - -class TestLocalSingerlake: - @pytest.fixture - def test_dir(self): - TEST_DIR.mkdir(exist_ok=True, parents=True) - yield TEST_DIR - shutil.rmtree(TEST_DIR) - - @pytest.fixture - def local_store(self, test_dir): - yield LocalStore(lake_root=test_dir) - - @pytest.fixture - def singerlake(self, local_store): - yield SingerLake(store=local_store) - - def test_write(self, singerlake): - for tap_id, streams in EXAMPLE_TAP_SYNC.items(): - for stream_id, stream in streams.items(): - stream_schema = {} - with stream["schema"].open() as schema_file: - stream_schema = json.load(schema_file) - singerlake.write_files( - tap_id=tap_id, - stream_id=stream_id, - stream_schema=stream_schema, - files=stream["files"], - ) - # check tap dir created - assert (LAKE_PATH / tap_id).exists() - # check stream dir created - assert (LAKE_PATH / tap_id / stream_id).exists() - # check manifest file created - assert (LAKE_PATH / tap_id / stream_id / "manifest.json").exists() - # check stream version dir created - assert (LAKE_PATH / tap_id / stream_id / stream["schema_hash"]).exists() - # check data file copied - for file in stream["files"]: - assert ( - LAKE_PATH - / tap_id - / stream_id - / stream["schema_hash"] - / file.name - ).exists() - assert True diff --git a/tests/test_local_store.py b/tests/test_local_store.py deleted file mode 100644 index a74586b..0000000 --- a/tests/test_local_store.py +++ /dev/null @@ -1,36 +0,0 @@ -import shutil -from pathlib import Path - -import pytest - -from singerlake.store import LocalStore - -TEST_DIR = Path(__file__).parent / "local_singerlake" / "test_local_store" - - -class TestLocalStore: - """Test Local Store.""" - - @pytest.fixture - def test_dir(self): - TEST_DIR.mkdir(exist_ok=True, parents=True) - yield TEST_DIR - shutil.rmtree(TEST_DIR) - - @pytest.fixture - def local_store(self, test_dir): - yield LocalStore(lake_root=test_dir) - - def test_dir_generators(self, local_store): - assert local_store.lake_root == TEST_DIR - assert local_store.get_lake_path() == TEST_DIR / "raw" - assert ( - local_store.get_tap_path(tap_id="tap-sample--meltano") - == TEST_DIR / "raw" / "tap-sample--meltano" - ) - assert ( - local_store.get_stream_path( - tap_id="tap-sample--meltano", stream_id="example-stream-1" - ) - == TEST_DIR / "raw" / "tap-sample--meltano" / "example-stream-1" - ) diff --git a/tests/test_singerlake.py b/tests/test_singerlake.py new file mode 100644 index 0000000..5d7d562 --- /dev/null +++ b/tests/test_singerlake.py @@ -0,0 +1,22 @@ +import pytest # noqa: F401 + + +def test_singerlake(singerlake): + assert singerlake.instance_id is not None + assert singerlake.config is not None + assert singerlake.store is not None + assert singerlake.manifest_service is not None + assert singerlake.discovery_service is not None + + +def test_discovery(singerlake): + assert singerlake.lake_id == "sound-oryx" + assert singerlake.list_taps() == ["tap-carbon-intensity"] + + tap = singerlake.get_tap("tap-carbon-intensity") + + assert tap.stream_ids == [ + "entry", + "generationmix", + "region", + ]