Add compactor performance regression checks within Github Workflows

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
         strategy:
             matrix:
                 python-version: ["3.7", "3.8", "3.9", "3.10"]
-        timeout-minutes: 10
+        timeout-minutes: 20
         steps:
             - name: "checkout repository"
               uses: actions/checkout@v3
@@ -47,7 +47,32 @@ jobs:
             - name: Install dependencies
               run: |
                   python -m pip install --upgrade pip
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-            - name: Run unit tests
+                  if [ -f benchmark-requirements.txt ]; then pip install -r benchmark-requirements.txt; fi
+            - name: Run unit tests + benchmarks
               run: >-
-                  python -m pytest
+                  python -m pytest --benchmark-json output.json
+            # Download previous benchmark result from cache (if exists)
+            - name: Download previous benchmark data
+              uses: actions/cache@v1
+              with:
+                path: ./cache
+                key: ${{ runner.os }}-benchmark
+            - name: Store benchmark results
+              uses: benchmark-action/github-action-benchmark@v1
+              with:
+                tool: 'pytest'
+                output-file-path: output.json
+                auto-push: false
+                github-token: ${{ secrets.GITHUB_TOKEN }}
+
+                # Where the previous data file is stored
+                external-data-json-path: ./cache/benchmark-data.json
+
+                # Enable Job Summary for PRs
+                summary-always: true
+
+                # Enable alert commit comment
+                #
+                # By default, this action marks the result as performance regression
+                # when it is worse than the previous exceeding 200% threshold.
+                comment-on-alert: true

.github/workflows/publish-to-pypi.yml

@@ -1,42 +1,42 @@
 name: Publish Python distributions to PyPI
 on:
-    release:
-        types: [published] # triggered whenever a new GitHub release is published
+  release:
+    types: [published] # triggered whenever a new GitHub release is published
 jobs:
-    build-n-publish:
-        name: Build and publish Python distributions to PyPI
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout
-              uses: actions/checkout@main
-              with:
-                  fetch-depth: 0
-            - name: Set up Python 3.7 (minimum supported python version for deltaCAT)
-              uses: actions/setup-python@v3
-              with:
-                  python-version: "3.7"
-            - name: Install pypa/build
-              run: >-
-                  python -m
-                  pip install
-                  build
-                  --user
-            - name: Install dependencies
-              run: |
-                  python -m pip install --upgrade pip
-                  pip install pytest
-                  if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-            - name: Run unit tests
-              run: >-
-                  python -m pytest
-            - name: Echo release tag
-              run: echo ${{ github.ref_name }}
-            - name: Build a binary wheel and a source tarball
-              run: >-
-                  python setup.py sdist bdist_wheel
-            - name: Publish distribution to PyPI
-              if: startsWith(github.ref, 'refs/tags')
-              uses: pypa/gh-action-pypi-publish@release/v1
-              with:
-                  password: ${{ secrets.PYPI_API_TOKEN }}
-                  verbose: true
+  build-n-publish:
+    name: Build and publish Python distributions to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.7 (minimum supported python version for deltaCAT)
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.7"
+      - name: Install pypa/build
+        run: >-
+          python -m
+          pip install
+          build
+          --user
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          if [ -f benchmark-requirements.txt ]; then pip install -r benchmark-requirements.txt; fi
+      - name: Run unit tests
+        run: >-
+          python -m pytest
+      - name: Echo release tag
+        run: echo ${{ github.ref_name }}
+      - name: Build a binary wheel and a source tarball
+        run: >-
+          python setup.py sdist bdist_wheel
+      - name: Publish distribution to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -6,6 +6,8 @@
 from typing import Any, Callable, Dict, List, Optional, Set
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
+from pytest_benchmark.fixture import BenchmarkFixture
+
 from deltacat.tests.compute.test_util_common import (
     get_rcf,
 )
@@ -161,6 +163,7 @@ def test_compact_partition_incremental(
     read_kwargs_provider_param: Any,
     skip_enabled_compact_partition_drivers,
     compact_partition_func: Callable,
+    benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
     from deltacat.types.media import ContentType
@@ -235,8 +238,22 @@ def test_compact_partition_incremental(
             "sort_keys": sort_keys if sort_keys else None,
         }
     )
+
     # execute
-    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+    def _incremental_compaction_setup():
+        """
+        This callable runs right before invoking the benchmark target function.
+        It ensures that each retry runs on a clean test environment by removing
+        any RCF generated from prior test runs.
+
+        Returns: args, kwargs
+        """
+        setup_s3_resource.Bucket(TEST_S3_RCF_BUCKET_NAME).objects.all().delete()
+        return (compact_partition_params,), {}
+
+    rcf_file_s3_uri = benchmark.pedantic(
+        compact_partition_func, setup=_incremental_compaction_setup
+    )
     # validate
     round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)
     compacted_delta_locator: DeltaLocator = (

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -5,6 +5,8 @@
 import boto3
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
+from pytest_benchmark.fixture import BenchmarkFixture
+
 from deltacat.tests.compute.test_util_constant import (
     BASE_TEST_SOURCE_NAMESPACE,
     BASE_TEST_SOURCE_TABLE_NAME,
@@ -182,6 +184,7 @@ def test_compact_partition_rebase_then_incremental(
     rebase_expected_compact_partition_result: pa.Table,
     skip_enabled_compact_partition_drivers,
     compact_partition_func: Callable,
+    benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
     from deltacat.types.media import ContentType
@@ -265,7 +268,7 @@ def test_compact_partition_rebase_then_incremental(
         }
     )
     # execute
-    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
     compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
         setup_s3_resource, rcf_file_s3_uri
     )