chore: improving codebase; adding semantic releases; fixing scripts & tests post migration

.github/workflows/cd.yaml

@@ -8,6 +8,11 @@ on:
         type: boolean
         required: true
         default: false
+      publish_github:
+        description: "Publish to Github?"
+        type: boolean
+        required: true
+        default: true
       run_checks:
         description: "Run checks?"
         type: boolean
@@ -63,7 +68,7 @@ jobs:
         run: hatch build
 
       - name: Run tests (codebase)
-        run: hatch run test:codebase
+        run: hatch run tests
 
       - name: Run tests (examples)
         run: hatch run examples:tests
@@ -75,8 +80,14 @@ jobs:
         with:
           path: dist
 
-      - name: Publish to PyPI - Unit Testing
-        if: ${{ !inputs.dry_run && inputs.publish_pypi }}
+      - name: Python Semantic Release
+        if: ${{ github.ref == 'refs/heads/main' && !inputs.dry_run && inputs.publish_github }}
+        uses: python-semantic-release/python-semantic-release@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish to PyPI
+        if: ${{ github.ref == 'refs/heads/main' && !inputs.dry_run && inputs.publish_pypi }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: dist

.github/workflows/ci.yaml

@@ -33,7 +33,7 @@ jobs:
         run: hatch build
 
       - name: Run tests (codebase)
-        run: hatch run test:codebase
+        run: hatch run tests
 
       - name: Run tests (examples)
         run: hatch run examples:tests

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: local
+    hooks:
+      - id: pre-commit
+        name: pre-commit
+        description: "Run pre-commit task via hatch"
+        entry: hatch run pre_commit
+        language: system
+        additional_dependencies: []
+        minimum_pre_commit_version: "0"
+
+      - id: examples-pre-commit
+        name: examples-pre-commit
+        description: "Run examples venv pre-commit task via hatch"
+        entry: hatch run examples:pre_commit
+        language: system
+        additional_dependencies: []
+        minimum_pre_commit_version: "0"

CONTRIBUTING.md

@@ -1,11 +1,14 @@
-# Development dependencies
+# Contributing to `algorand-python-testing`
+
+Welcome to the Algorand Python Testing library! This guide will help you get started with the project and contribute to its development.
 
 # Development Dependencies
 
 Our development environment relies on the following tools:
 
 -   **Python 3.12**
 -   **[Hatch](https://hatch.pypa.io/1.9/install/)**: A modern, extensible Python project manager.
+-   **[pre-commit](https://pre-commit.com/)**: A framework for managing and maintaining code quality.
 
 ## Common Commands
 
@@ -22,6 +25,12 @@ Here are some common commands you will use with Hatch:
 -   **Regenerate typed clients for example contracts:** `hatch run refresh_test_artifacts`
 -   **Coverage check of existing progress on implementing AlgoPy Stubs:** `hatch run check_stubs_cov`
 
+# Using `pre-commit`
+
+Execute `pre-commit install` to ensure auto run of hatch against `src` and `examples` folders
+
 # Examples folder
 
 Examples folder uses a dedicated 'venv.examples' virtual environment managed by Hatch that simulates a user environment with both algorand-python and algorand-python-testing installed explicitly. This is useful for testing new features or bug fixes in the testing library.
+
+-   **Pre-commit checks against examples:** `hatch run examples:pre-commit`

pyproject.toml

@@ -52,7 +52,7 @@ type = "virtual"
 path = ".venv"
 python = "3.12"
 dependencies = [
-    "puyapy>=2.1",
+    "puyapy>=2.1.2",
     "pytest>=7.4",
     "pytest-mock>=3.10.0",
     "pytest-xdist[psutil]>=3.3",
@@ -81,6 +81,7 @@ check = [
 ]
 # type checks algorand-python-testing code
 mypy_testing = "mypy . --exclude examples"
+tests = "pytest --cov --cov-report xml"
 
 # isolated environments for dev tooling
 [tool.hatch.envs.lint]
@@ -109,6 +110,7 @@ fix_examples = [ 'black examples', 'ruff check --fix examples']
 path = ".venv.cicd"
 dependencies = [
     "algokit",
+    "python-semantic-release>=9.8.5",
 ]
 [tool.hatch.envs.cicd.scripts]
 localnet_start = [
@@ -158,17 +160,6 @@ check = [
     "hatch run mypy examples",
 ]
 
-[tool.hatch.envs.test]
-dependencies = [
-    "pytest>=7.4",
-    "pytest-cov>=4.1.0",
-]
-[tool.hatch.envs.test.scripts]
-codebase = "pytest --cov --cov-report xml"
-
-[tool.hatch.envs.test.env-vars]
-PYTHONPATH = "./src"
-
 # tool configurations
 [tool.black]
 line-length = 99
@@ -277,3 +268,29 @@ follow_imports = "skip"
 
 [tool.pytest.ini_options]
 addopts = "-n auto --cov-config=.coveragerc"
+pythonpath = ['src']
+
+# === Semantic releases config ===
+
+[tool.semantic_release]
+version_toml = ["pyproject.toml:project.version"]
+commit_message = "{version}\n\nAutomatically generated by python-semantic-release"
+tag_format = "v{version}"
+major_on_zero = true
+
+[tool.semantic_release.branches.main]
+match = "main"
+prerelease_token = "beta"
+prerelease = false
+
+[tool.semantic_release.commit_parser_options]
+allowed_tags = ["build", "chore", "ci", "docs", "feat", "fix", "perf", "style", "refactor", "test"]
+minor_tags = ["feat"]
+patch_tags = ["fix", "perf", "docs"]
+
+[tool.semantic_release.publish]
+dist_glob_patterns = ["dist/*"] # order here is important to ensure compiler wheel is published first
+upload_to_vcs_release = true
+
+[tool.semantic_release.remote.token]
+env = "GITHUB_TOKEN"

scripts/check_stubs_cov.py

@@ -1,17 +1,16 @@
 import ast
 import inspect
+import site
 import sys
 from collections.abc import Iterable
 from pathlib import Path
 from typing import NamedTuple
 
 from prettytable import PrettyTable
 
-# TODO: update to follow the new project structure
-
-PROJECT_ROOT = (Path(__file__).parent / "..").resolve()
-VCS_ROOT = (PROJECT_ROOT / "..").resolve()
-STUBS = VCS_ROOT / "stubs" / "algopy-stubs"
+PROJECT_ROOT = Path(__file__).parent.parent.resolve()
+SITE_PACKAGES = Path(site.getsitepackages()[0])
+STUBS_ROOT = SITE_PACKAGES / "algopy-stubs"
 IMPL = PROJECT_ROOT / "src"
 ROOT_MODULE = "algopy"
 
@@ -49,13 +48,20 @@ def coverage(self) -> float:
 
 
 def main() -> None:
+    clear_algopy_content()
     stubs = collect_public_stubs()
     coverage = collect_coverage(stubs)
     print_results(coverage)
 
 
+def clear_algopy_content() -> None:
+    algopy_path = SITE_PACKAGES / "algopy.py"
+    if algopy_path.exists():
+        algopy_path.write_text("")
+
+
 def collect_public_stubs() -> dict[str, ASTNodeDefinition]:
-    stubs_root = STUBS / "__init__.pyi"
+    stubs_root = STUBS_ROOT / "__init__.pyi"
     stubs_ast = _parse_python_file(stubs_root)
     result = dict[str, ASTNodeDefinition]()
     for stmt in stubs_ast.body:
@@ -80,15 +86,15 @@ def collect_imports(
     dest = alias.asname
     # from module import *
     if src == "*" and dest is None:
-        for defn_name, defn in collect_stubs(STUBS, relative_module).items():
+        for defn_name, defn in collect_stubs(STUBS_ROOT, relative_module).items():
             yield f"{ROOT_MODULE}.{defn_name}", defn
     # from root import src as src
     elif stmt.module == ROOT_MODULE and dest == src:
-        for defn_name, defn in collect_stubs(STUBS, src).items():
+        for defn_name, defn in collect_stubs(STUBS_ROOT, src).items():
             yield f"{ROOT_MODULE}.{dest}.{defn_name}", defn
     # from foo.bar import src as src
     elif dest == src:
-        stubs = collect_stubs(STUBS, relative_module)
+        stubs = collect_stubs(STUBS_ROOT, relative_module)
         yield f"{ROOT_MODULE}.{src}", stubs[src]
     else:
         raise NotImplementedError
@@ -138,7 +144,7 @@ def collect_coverage(stubs: dict[str, ASTNodeDefinition]) -> list[CoverageResult
         result.append(
             CoverageResult(
                 full_name=full_name,
-                stub_file=str(stub.path.relative_to(STUBS)),
+                stub_file=str(stub.path.relative_to(STUBS_ROOT)),
                 impl_file=impl_file,
                 coverage=coverage.coverage if coverage else 0,
                 missing=", ".join(coverage.missing if coverage else []),
@@ -185,15 +191,25 @@ def _parse_python_file(filepath: Path) -> ast.Module:
 
 def _get_impl_coverage(symbol: str, stub: ASTNodeDefinition) -> ImplCoverage | None:
     import importlib
+    import sys
+    from pathlib import Path
+
+    # Add the src directory to the Python path
+    src_path = Path(__file__).parent.parent / "src"
+    sys.path.insert(0, str(src_path))
 
     module, name = symbol.rsplit(".", maxsplit=1)
     try:
+        # Use importlib.import_module for both algopy and non-algopy modules
         mod = importlib.import_module(module)
-    except ImportError:
+    except ImportError as ex:
+        print(f"Error importing {module}: {ex}")
         return None
+
     try:
         impl = getattr(mod, name)
     except AttributeError:
+        print(f"Attribute {name} not found in module {module}")
         return None
 
     try:

scripts/refresh_test_artifacts.py

@@ -27,6 +27,8 @@ def compile_contract(folder: Path) -> None:
         "hatch",
         "run",
         "puyapy",
+        "--log-level",
+        "debug",
         str(contract_path),
         "--out-dir",
         "data",

tests/artifacts/Arc4PrimitiveOps/data/Arc4PrimitiveOpsContract.approval.teal

@@ -876,7 +876,7 @@ main_verify_ufixednxm_bytes_route@32:
     dup
     len
     itob
-    substring 6 8
+    extract 6 2
     swap
     concat
     byte 0x151f7c75
@@ -903,7 +903,7 @@ main_verify_bigufixednxm_bytes_route@33:
     dup
     len
     itob
-    substring 6 8
+    extract 6 2
     swap
     concat
     byte 0x151f7c75
@@ -1089,7 +1089,7 @@ main_verify_string_bytes_route@41:
     dup
     len
     itob
-    substring 6 8
+    extract 6 2
     swap
     concat
     byte 0x151f7c75
@@ -1160,7 +1160,7 @@ main_verify_bool_bytes_route@44:
     dup
     len
     itob
-    substring 6 8
+    extract 6 2
     swap
     concat
     byte 0x151f7c75
@@ -1243,21 +1243,24 @@ main_verify_emit_route@47:
     dup
     int 0
     extract_uint16
-    dup2
-    extract_uint16
+    swap
+    dup
     int 2
-    +
-    extract3
+    extract_uint16
+    swap
+    cover 2
+    substring3
     extract 2 0
     txna ApplicationArgs 15
     dup
     int 2
     extract_uint16
-    dup2
-    extract_uint16
-    int 2
-    +
-    extract3
+    swap
+    dup
+    len
+    swap
+    cover 2
+    substring3
     extract 2 0
     // tests/artifacts/Arc4PrimitiveOps/contract.py:252
     // @arc4.abimethod()
@@ -2232,7 +2235,7 @@ verify_string_add:
     dup
     len
     itob
-    extract 6 0
+    extract 6 2
     swap
     concat
     // tests/artifacts/Arc4PrimitiveOps/contract.py:221
@@ -2349,7 +2352,7 @@ verify_bool_from_log:
 verify_emit:
     // tests/artifacts/Arc4PrimitiveOps/contract.py:252-271
     // @arc4.abimethod()
-    // def verify_emit(
+    // def verify_emit(  # noqa: PLR0913
     //     self,
     //     a: arc4.String,
     //     b: arc4.UInt512,