diff --git a/.github/labeler.yml b/.github/labeler.yml index ad166870..c7511fcc 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,27 +1,41 @@ area/workflow/dotnet_clipackage: -- aws_lambda_builders/workflows/dotnet_clipackage/* -- aws_lambda_builders/workflows/dotnet_clipackage/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/dotnet_clipackage/* + - aws_lambda_builders/workflows/dotnet_clipackage/**/* area/workflow/go_modules: -- aws_lambda_builders/workflows/go_modules/* -- aws_lambda_builders/workflows/go_modules/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/go_modules/* + - aws_lambda_builders/workflows/go_modules/**/* area/workflow/java_gradle: -- aws_lambda_builders/workflows/java_gradle/* -- aws_lambda_builders/workflows/java_gradle/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/java_gradle/* + - aws_lambda_builders/workflows/java_gradle/**/* area/workflow/java_maven: -- aws_lambda_builders/workflows/java_maven/* -- aws_lambda_builders/workflows/java_maven/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/java_maven/* + - aws_lambda_builders/workflows/java_maven/**/* area/workflow/node_npm: -- aws_lambda_builders/workflows/nodejs_npm/* -- aws_lambda_builders/workflows/nodejs_npm/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/nodejs_npm/* + - aws_lambda_builders/workflows/nodejs_npm/**/* area/workflow/python_pip: -- aws_lambda_builders/workflows/python_pip/* -- aws_lambda_builders/workflows/python_pip/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/python_pip/* + - aws_lambda_builders/workflows/python_pip/**/* area/workflow/ruby_bundler: -- aws_lambda_builders/workflows/ruby_bundler/* -- aws_lambda_builders/workflows/ruby_bundler/**/* + - changed-files: + - any-glob-to-any-file: + - aws_lambda_builders/workflows/ruby_bundler/* + - aws_lambda_builders/workflows/ruby_bundler/**/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 192eb668..b912f095 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: - "3.10" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Unit Testing @@ -97,7 +97,7 @@ jobs: - 9 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - uses: actions/setup-node@v4 @@ -131,7 +131,7 @@ jobs: - 9 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - uses: actions/setup-node@v4 @@ -162,10 +162,10 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: '^1.16' - run: make init @@ -190,10 +190,10 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'corretto' java-version: '21' @@ -221,10 +221,10 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: '21' @@ -250,7 +250,7 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: make init @@ -275,7 +275,7 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: | @@ -304,7 +304,7 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - uses: ruby/setup-ruby@v1 @@ -335,7 +335,7 @@ jobs: - "3.12" steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: make init @@ -364,7 +364,7 @@ jobs: - stable steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/close-issue-on-release.yml b/.github/workflows/close-issue-on-release.yml new file mode 100644 index 00000000..413b6cf9 --- /dev/null +++ b/.github/workflows/close-issue-on-release.yml @@ -0,0 +1,21 @@ +name: Close issues on release cut + +on: + release: + types: [released] + +jobs: + run-workflow: + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Close issues marked + env: + REPO : ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + release_url=$(gh release view ${{ github.ref_name }} --repo $REPO --json url --jq ".url") + for issue_number in $(gh issue list -l "stage/waiting-for-release" --repo $REPO --json number --jq ".[].number"); do + gh issue close $issue_number -c "Patch is released in [${{ github.ref_name }}]($release_url). If you are AWS SAM CLI user, please wait for next AWS SAM CLI release. Closing" -r completed --repo $REPO + done \ No newline at end of file diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 0567b1a2..dc2a77e4 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -9,7 +9,7 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" apply-internal-external-label: diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index c35fe4e1..32702e3b 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -5,5 +5,5 @@ # Changing version will trigger a new release! # Please make the version change as the last step of your development. -__version__ = "1.43.0" +__version__ = "1.44.0" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index d325559d..dcef47a9 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -49,11 +49,18 @@ def execute(self) -> None: pip_runner = PipRunner(python_exe=python_with_pip, pip=pip) dependency_builder = DependencyBuilder( - osutils=self._os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture + osutils=self._os_utils, + python_exe=python_with_pip, + pip_runner=pip_runner, + runtime=self.runtime, + architecture=self.architecture, ) package_builder = PythonPipDependencyBuilder( - osutils=self._os_utils, runtime=self.runtime, dependency_builder=dependency_builder + osutils=self._os_utils, + runtime=self.runtime, + python_exe=python_with_pip, + dependency_builder=dependency_builder, ) try: target_artifact_dir = self.artifacts_dir diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 252e45f7..575ecb7d 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -5,7 +5,6 @@ import logging import re import subprocess -import sys from email.parser import FeedParser from typing import Tuple @@ -97,7 +96,7 @@ def get_lambda_abi(runtime): class PythonPipDependencyBuilder(object): - def __init__(self, runtime, osutils=None, dependency_builder=None, architecture=X86_64): + def __init__(self, runtime, python_exe, osutils=None, dependency_builder=None, architecture=X86_64): """Initialize a PythonPipDependencyBuilder. :type runtime: str @@ -122,7 +121,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None, architecture= self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils, runtime, architecture=architecture) + dependency_builder = DependencyBuilder(self.osutils, python_exe, runtime, architecture=architecture) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_path, ui=None, config=None): @@ -211,7 +210,7 @@ class DependencyBuilder(object): # Unlikely to hit this case. _DEFAULT_GLIBC = (2, 17) - def __init__(self, osutils, runtime, pip_runner=None, architecture=X86_64): + def __init__(self, osutils, runtime, python_exe, pip_runner=None, architecture=X86_64): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -229,8 +228,9 @@ def __init__(self, osutils, runtime, pip_runner=None, architecture=X86_64): :param architecture: Architecture to build for. """ self._osutils = osutils + self.python_exe = python_exe if pip_runner is None: - pip_runner = PipRunner(python_exe=None, pip=SubprocessPip(osutils)) + pip_runner = PipRunner(python_exe=python_exe, pip=SubprocessPip(osutils)) self._pip = pip_runner self.runtime = runtime self.architecture = architecture @@ -364,7 +364,10 @@ def _download_all_dependencies(self, requirements_filename, directory): # which will serve as the primary list of dependencies needed to deploy # successfully. self._pip.download_all_dependencies(requirements_filename, directory) - deps = {Package(directory, filename) for filename in self._osutils.get_directory_contents(directory)} + deps = { + Package(directory, filename, self.python_exe) + for filename in self._osutils.get_directory_contents(directory) + } LOG.debug("Full dependency closure: %s", deps) return deps @@ -383,7 +386,7 @@ def _build_sdists(self, sdists, directory, compile_c=True): def _categorize_wheel_files(self, directory): final_wheels = [ - Package(directory, filename) + Package(directory, filename, self.python_exe) for filename in self._osutils.get_directory_contents(directory) if filename.endswith(".whl") ] @@ -506,13 +509,14 @@ def _install_wheels(self, src_dir, dst_dir, wheels): class Package(object): """A class to represent a package downloaded but not yet installed.""" - def __init__(self, directory, filename, osutils=None): + def __init__(self, directory, filename, python_exe, osutils=None): self.dist_type = "wheel" if filename.endswith(".whl") else "sdist" self._directory = directory self.filename = filename if osutils is None: osutils = OSUtils() self._osutils = osutils + self.python_exe = python_exe self._name, self._version = self._calculate_name_and_version() @property @@ -553,7 +557,7 @@ def _calculate_name_and_version(self): # {platform tag}.whl name, version = self.filename.split("-")[:2] else: - info_fetcher = SDistMetadataFetcher(osutils=self._osutils) + info_fetcher = SDistMetadataFetcher(self.python_exe, osutils=self._osutils) sdist_path = self._osutils.joinpath(self._directory, self.filename) name, version = info_fetcher.get_package_name_and_version(sdist_path) normalized_name = self._normalize_name(name) @@ -572,10 +576,11 @@ class SDistMetadataFetcher(object): "exec(compile(code, __file__, 'exec'))" ) - def __init__(self, osutils=None): + def __init__(self, python_exe, osutils=None): if osutils is None: osutils = OSUtils() self._osutils = osutils + self.python_exe = python_exe def _parse_pkg_info_file(self, filepath): # The PKG-INFO generated by the egg-info command is in an email feed @@ -590,7 +595,7 @@ def _get_pkg_info_filepath(self, package_dir): setup_py = self._osutils.joinpath(package_dir, "setup.py") script = self._SETUPTOOLS_SHIM % setup_py - cmd = [sys.executable, "-c", script, "--no-user-cfg", "egg_info", "--egg-base", "egg-info"] + cmd = [self.python_exe, "-c", script, "--no-user-cfg", "egg_info", "--egg-base", "egg-info"] egg_info_dir = self._osutils.joinpath(package_dir, "egg-info") self._osutils.makedirs(egg_info_dir) p = subprocess.Popen( diff --git a/requirements/dev.txt b/requirements/dev.txt index 20af74f1..a5faa70b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -14,4 +14,4 @@ pyelftools~=0.30 # Used to verify the generated Go binary architecture in integr # formatter black==22.6.0; python_version < "3.8" black==23.11.0; python_version >= "3.8" -ruff==0.1.5 +ruff==0.1.7 diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 615d5e60..2a674afe 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -33,7 +33,8 @@ def _create_app_structure(tmpdir): @pytest.fixture def sdist_reader(): - return SDistMetadataFetcher() + # We are removing references to sys.executable from the business logic but are using it here for testing purposes + return SDistMetadataFetcher(python_exe=sys.executable) @pytest.fixture @@ -141,7 +142,7 @@ def __init__(self, filename, dirarg, expected_args, whl_contents=None, expected_ def _build_fake_whl(self, directory, filename): filepath = os.path.join(directory, filename) if not os.path.isfile(filepath): - package = Package(directory, filename) + package = Package(directory, filename, python_exe=sys.executable) with zipfile.ZipFile(filepath, "w") as z: for content_path in self._whl_contents: z.writestr(content_path.format(package_name=self._package_name, data_dir=package.data_dir), b"") @@ -204,7 +205,7 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, **kwargs): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), "python3.9", runner, **kwargs) + builder = DependencyBuilder(OSUtils(), "python3.9", sys.executable, runner, **kwargs) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -1020,22 +1021,22 @@ def test_same_pkg_sdist_and_wheel_collide(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: sdist_builder.write_fake_sdist(tempdir, "foobar", "1.0") pkgs = set() - pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) - pkgs.add(Package(tempdir, "foobar-1.0.zip")) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable)) + pkgs.add(Package(tempdir, "foobar-1.0.zip", python_exe=sys.executable)) assert len(pkgs) == 1 def test_ensure_sdist_name_normalized_for_comparison(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: sdist_builder.write_fake_sdist(tempdir, "Foobar", "1.0") pkgs = set() - pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) - pkgs.add(Package(tempdir, "Foobar-1.0.zip")) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable)) + pkgs.add(Package(tempdir, "Foobar-1.0.zip", python_exe=sys.executable)) assert len(pkgs) == 1 def test_ensure_wheel_name_normalized_for_comparison(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: sdist_builder.write_fake_sdist(tempdir, "foobar", "1.0") pkgs = set() - pkgs.add(Package("", "Foobar-1.0-py3-none-any.whl")) - pkgs.add(Package(tempdir, "foobar-1.0.zip")) + pkgs.add(Package("", "Foobar-1.0-py3-none-any.whl", python_exe=sys.executable)) + pkgs.add(Package(tempdir, "foobar-1.0.zip", python_exe=sys.executable)) assert len(pkgs) == 1 diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 2917fdf4..8cd43c04 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -30,7 +30,9 @@ def test_action_must_call_builder(self, find_runtime_mock, dependency_builder_mo ) action.execute() - dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64) + dependency_builder_mock.assert_called_with( + osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64, python_exe=ANY + ) builder_instance.build_dependencies.assert_called_with( artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" @@ -56,7 +58,9 @@ def test_action_must_call_builder_with_architecture( ) action.execute() - dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64) + dependency_builder_mock.assert_called_with( + osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64, python_exe=ANY + ) builder_instance.build_dependencies.assert_called_with( artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 5b8db021..e2d4300b 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,7 +1,7 @@ import sys from collections import namedtuple from unittest import TestCase, mock -from unittest.mock import patch +from unittest.mock import ANY, patch import pytest from parameterized import parameterized @@ -115,7 +115,7 @@ def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) osutils_mock = mock.Mock(spec=osutils) builder = PythonPipDependencyBuilder( - osutils=osutils_mock, dependency_builder=mock_dep_builder, runtime="runtime" + osutils=osutils_mock, dependency_builder=mock_dep_builder, runtime="runtime", python_exe=sys.executable ) builder.build_dependencies("artifacts/path/", "scratch_dir/path/", "path/to/requirements.txt") mock_dep_builder.build_site_packages.assert_called_once_with( @@ -125,24 +125,26 @@ def test_can_call_dependency_builder(self, osutils): @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") def test_can_create_new_dependency_builder(self, DependencyBuilderMock, osutils): osutils_mock = mock.Mock(spec=osutils) - builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime") - DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=X86_64) + builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime", python_exe=sys.executable) + DependencyBuilderMock.assert_called_with(osutils_mock, ANY, "runtime", architecture=X86_64) @mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") def test_can_call_dependency_builder_with_architecture(self, DependencyBuilderMock, osutils): osutils_mock = mock.Mock(spec=osutils) - builder = PythonPipDependencyBuilder(osutils=osutils_mock, runtime="runtime", architecture=ARM64) - DependencyBuilderMock.assert_called_with(osutils_mock, "runtime", architecture=ARM64) + builder = PythonPipDependencyBuilder( + osutils=osutils_mock, runtime="runtime", architecture=ARM64, python_exe=sys.executable + ) + DependencyBuilderMock.assert_called_with(osutils_mock, ANY, "runtime", architecture=ARM64) class TestPackage(object): def test_can_create_package_with_custom_osutils(self, osutils): - pkg = Package("", "foobar-1.0-py3-none-any.whl", osutils) + pkg = Package("", "foobar-1.0-py3-none-any.whl", sys.executable, osutils) assert pkg._osutils == osutils def test_wheel_package(self): filename = "foobar-1.0-py3-none-any.whl" - pkg = Package("", filename) + pkg = Package("", filename, python_exe=sys.executable) assert pkg.dist_type == "wheel" assert pkg.filename == filename assert pkg.identifier == "foobar==1.0" @@ -150,42 +152,42 @@ def test_wheel_package(self): def test_invalid_package(self): with pytest.raises(InvalidSourceDistributionNameError): - Package("", "foobar.jpg") + Package("", "foobar.jpg", python_exe=sys.executable) def test_diff_pkg_sdist_and_whl_do_not_collide(self): pkgs = set() - pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) - pkgs.add(Package("", "badbaz-1.0-py3-none-any.whl")) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable)) + pkgs.add(Package("", "badbaz-1.0-py3-none-any.whl", python_exe=sys.executable)) assert len(pkgs) == 2 def test_same_pkg_is_eq(self): - pkg = Package("", "foobar-1.0-py3-none-any.whl") + pkg = Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable) assert pkg == pkg def test_pkg_is_eq_to_similar_pkg(self): - pure_pkg = Package("", "foobar-1.0-py3-none-any.whl") - plat_pkg = Package("", "foobar-1.0-py3-py39-manylinux1_x86_64.whl") + pure_pkg = Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable) + plat_pkg = Package("", "foobar-1.0-py3-py39-manylinux1_x86_64.whl", python_exe=sys.executable) assert pure_pkg == plat_pkg def test_pkg_is_not_equal_to_different_type(self): - pkg = Package("", "foobar-1.0-py3-none-any.whl") + pkg = Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable) non_package_type = 1 assert not (pkg == non_package_type) def test_pkg_repr(self): - pkg = Package("", "foobar-1.0-py3-none-any.whl") + pkg = Package("", "foobar-1.0-py3-none-any.whl", python_exe=sys.executable) assert repr(pkg) == "foobar==1.0(wheel)" def test_wheel_data_dir(self): - pkg = Package("", "foobar-2.0-py3-none-any.whl") + pkg = Package("", "foobar-2.0-py3-none-any.whl", python_exe=sys.executable) assert pkg.data_dir == "foobar-2.0.data" def test_can_read_packages_with_underscore_in_name(self): - pkg = Package("", "foo_bar-2.0-py3-none-any.whl") + pkg = Package("", "foo_bar-2.0-py3-none-any.whl", python_exe=sys.executable) assert pkg.identifier == "foo-bar==2.0" def test_can_read_packages_with_period_in_name(self): - pkg = Package("", "foo.bar-2.0-py3-none-any.whl") + pkg = Package("", "foo.bar-2.0-py3-none-any.whl", python_exe=sys.executable) assert pkg.identifier == "foo-bar==2.0"