Build with Poetry (#1)

.flake8

@@ -0,0 +1,11 @@
+[flake8]
+ignore = E203, E266, W503, BLK100, W291, I004
+max-line-length = 120
+max-complexity = 15
+
+[isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+line_length = 120

.github/workflows/mega-linter.yml

@@ -0,0 +1,56 @@
+---
+# MegaLinter GitHub Action configuration file
+# More info at https://megalinter.github.io
+name: MegaLinter
+on:  # yamllint disable-line rule:truthy
+  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
+permissions: read-all
+env: # Comment env block if you do not want to apply fixes
+  APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
+  DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,MAKEFILE_CHECKMAKE,PYTHON_BANDIT,PYTHON_PYRIGHT,PYTHON_PYLINT,REPOSITORY_GRYPE,REPOSITORY_SECRETLINT,REPOSITORY_TRIVY,REPOSITORY_TRUFFLEHOG,REPOSITORY_CHECKOV
+  MARKDOWN_MARKDOWNLINT_FILTER_REGEX_EXCLUDE: "tests/example.*ME\\.md" # Exclude example markdown files from markdownlint
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: MegaLinter
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+      issues: write
+      pull-requests: write
+    steps:
+      # Git Checkout
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances
+
+      # MegaLinter
+      - name: MegaLinter
+        id: ml
+        # You can override MegaLinter flavor used to have faster performances
+        # More info at https://megalinter.github.io/flavors/
+        uses: oxsecurity/megalinter/flavors/[email protected]
+        env:
+          # All available variables are described in documentation
+          # https://megalinter.github.io/configuration/
+          VALIDATE_ALL_CODEBASE: true  
+          # VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Validates all source when push on main, else just the git diff with main. Override with true if you always want to lint all sources
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY
+          DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks
+
+      # Upload MegaLinter artifacts
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v4
+        if: true
+        with:
+          name: MegaLinter reports
+          path: |
+            megalinter-reports
+            mega-linter.log

.github/workflows/python-package.yml

@@ -0,0 +1,52 @@
+---
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+name: Python package
+on:  # yamllint disable-line rule:truthy
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+permissions: read-all
+
+jobs:
+  build:
+    permissions:
+      contents: read
+      id-token: write
+      issues: write
+      pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+        poetry-version: ["1.8.3"]
+        os: [ubuntu-latest, macos-latest] # , windows-latest
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Run poetry image
+      uses: abatilo/actions-poetry@v2
+      with:
+        poetry-version: ${{ matrix.poetry-version }}
+    - name: Install dependencies
+      run: |
+        poetry --version
+        poetry install
+    - name: Test with pytest
+      run: |
+        make test TEST_OS=${{ matrix.os }}
+    - name: Get Coverage Report
+      uses: orgoro/[email protected]
+      with:
+          coverageFile: coverage.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          thresholdAll: 0.8
+          thresholdNew: 0.8
+          thresholdModified: 0.8
+      if: github.event_name == 'pull_request'

.gitignore

@@ -8,3 +8,6 @@ dist/
 
 # Environment info
 .env
+athena_federation/.DS_Store
+coverage.xml
+.coverage

.vscode/settings.json

@@ -3,5 +3,6 @@
     "python.pythonPath": ".venv/bin/python",
     "python.analysis.extraPaths": [
         "src.*"
-    ]
+    ],
+    "python.analysis.typeCheckingMode": "basic"
 }

CHANGELOG.md

@@ -0,0 +1,35 @@
+# CHANGELOG.md
+
+## [0.1.6] - 2024-08-24
+
+- Move py.typed into package folder
+
+## [0.1.5] - 2024-08-24
+
+- Add py.typed file to support type checking
+- Add `mypy` to `make lint`
+- Fix type errors
+- Add helpers
+
+## [0.1.4] - 2024-08-23
+
+- Update dependencies
+
+## [0.1.3] - 2024-08-19
+
+- Fixes suggested by upstream
+
+## [0.1.2] - 2024-08-18
+
+- Support Python 3.13
+- add flake8 to linting
+
+## [0.1.1] - 2024-08-17
+
+- Fix Dockerfile to use `poetry install` instead of `pip install`
+
+## [0.1.0] - 2024-08-12
+
+- Use Poetry
+- Add CI tests
+- Create Python package

Dockerfile

@@ -1,25 +1,29 @@
-FROM python:3.8-slim AS build
-
-# Set our workdir
+# adapted from https://dev.to/farcellier/package-a-poetry-project-in-a-docker-container-for-production-3b4m
+# and  https://medium.com/@albertazzir/blazing-fast-python-docker-builds-with-poetry-a78a66f5aed0
+FROM amazon/aws-lambda-python:3.12 AS build
 WORKDIR /app
 
 # Get ready to build
-RUN pip install build
+RUN pip install --no-cache-dir poetry==1.8.3
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
 
-# Now copy the app over and build a wheel
-COPY src /app/src
-COPY pyproject.toml setup.cfg /app/
-RUN python -m build
+COPY pyproject.toml poetry.lock ./
+RUN touch /app/README.md
+COPY athena_federation ./athena_federation
+RUN poetry build -f wheel
 
-## Now use the compiled wheel in our lambda function
-FROM amazon/aws-lambda-python:3.8.2021.12.09.15 AS lambda
+FROM amazon/aws-lambda-python:3.12 AS lambda
+WORKDIR /app
 
-ENV TARGET_BUCKET=replace_me
+ENV TARGET_BUCKET=quilt-example
 
-COPY --from=build /app/dist/unoffical_athena_federation_sdk-*-py3-none-any.whl /
-RUN pip install /unoffical_athena_federation_sdk-*-py3-none-any.whl
+COPY --from=build /app/dist/athena_federation-*-py3-none-any.whl /
+RUN pip install --no-cache-dir /athena_federation-*-py3-none-any.whl
 
-COPY example/ ./
-RUN ls ./
+COPY example  ${LAMBDA_TASK_ROOT}/example
 
-CMD [ "handler.lambda_handler" ]
+CMD [ "example.handler.sample_handler" ]

Makefile

@@ -1,6 +1,112 @@
-build:
-	python3 -m build
-upload:
-	python3 -m twine upload dist/*
+.PHONY : help install update test lint clean build publish all
 
-.PHONY: build upload
+#
+# Variables
+#
+
+PROJ = athena_federation
+CONTAINER = athena-federation
+VERSION = $(shell poetry version | cut -d' ' -f2)
+IMG = local/$(CONTAINER):$(VERSION)
+CLIENT_PORT = 9000
+SERVER_PORT = 8080
+PORTMAP = -p $(CLIENT_PORT):$(SERVER_PORT)
+ARCH = amd64  # force to standard platform
+PLAT = --platform=linux/$(ARCH)
+BUILD = image build $(PLAT)
+RUN = container run $(PLAT)
+
+#
+# Development
+#
+
+all: test
+
+lock:
+	poetry lock
+
+# Install project dependencies
+install: lock
+	poetry install
+
+# Update project dependencies
+update:
+	poetry update
+
+# Run project tests
+test:
+	poetry run pytest --cov-report xml --cov=athena_federation
+
+coverage:
+	poetry run pytest --cov-report html --cov=athena_federation
+	open htmlcov/index.html
+
+watch:
+	poetry run ptw . --now
+
+# Lint code using flake8
+lint:
+	poetry run black .
+	poetry run flake8
+	poetry run mypy .
+
+# Clean up generated files
+clean:
+	poetry run rm -rf dist build *cov* .*cache
+
+# Build project distribution
+build: lint
+	poetry check
+	poetry build
+
+# Publish project to PyPI
+publish: clean test build
+	poetry publish
+
+#
+# Docker Commands
+#
+
+docker: docker-stop docker-build docker-detached lambda-ping lambda-list-schemas
+
+# Verify Docker is running
+
+docker-status:
+	docker ps
+
+# Build Docker image
+docker-build:
+	docker $(BUILD) -t $(IMG) .
+	docker images | grep $(CONTAINER)
+
+docker-debug:
+	docker $(BUILD) . --no-cache --build-arg DEBUG=true
+
+# One blog post claims this is necessary to get poetry to work in a docker container
+docker-poetry-config:
+	poetry config virtualenvs.in-project true --local
+
+# Run Docker container
+
+docker-run: docker-build
+	docker $(RUN) --rm $(PORTMAP) $(IMG)
+
+# Run Docker container in detached mode (untested)
+
+docker-detached:
+	docker $(RUN) -d  $(PORTMAP) $(IMG)
+	sleep 5
+
+# Stop Docker container
+
+docker-stop:
+	docker ps -q | xargs --no-run-if-empty docker stop
+
+
+# Ping lambda
+
+lambda-ping:
+	curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"@type": "PingRequest", "identity": {"id": "UNKNOWN", "principal": "UNKNOWN", "account": "123456789012", "arn": "arn:aws:iam::123456789012:root", "tags": {}, "groups": []}, "catalogName": "athena_python_sdk", "queryId": "1681559a-548b-4771-874c-2aa2ea7c39ab"}'
+
+lambda-list-schemas:
+	curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"@type": "ListSchemasRequest", "identity": {"id": "UNKNOWN", "principal": "UNKNOWN", "account": "123456789012", "arn": "arn:aws:iam::123456789012:root", "tags": {}, "groups": []}, "catalogName": "athena_python_sdk", "queryId": "1681559a-548b-4771-874c-2aa2ea7c39ab"}'