From 14e7f10456f786ebfbc7c3d0e0d65163f26516e0 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+TheSriram@users.noreply.github.com> Date: Tue, 12 Mar 2019 14:20:29 -0700 Subject: [PATCH 1/4] fix: maven builder logs (#97) --- aws_lambda_builders/workflows/java_maven/maven.py | 12 ++++++------ tests/unit/workflows/java_maven/test_maven.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/aws_lambda_builders/workflows/java_maven/maven.py b/aws_lambda_builders/workflows/java_maven/maven.py index f17ab0591..dd778f792 100644 --- a/aws_lambda_builders/workflows/java_maven/maven.py +++ b/aws_lambda_builders/workflows/java_maven/maven.py @@ -28,25 +28,25 @@ def __init__(self, maven_binary, os_utils=None): def retrieve_module_name(self, scratch_dir): args = ['-q', '-Dexec.executable=echo', '-Dexec.args=${project.artifactId}', 'exec:exec', '--non-recursive'] - ret_code, stdout, stderr = self._run(args, scratch_dir) + ret_code, stdout, _ = self._run(args, scratch_dir) if ret_code != 0: - raise MavenExecutionError(message=stderr.decode('utf8').strip()) + raise MavenExecutionError(message=stdout.decode('utf8').strip()) return stdout.decode('utf8').strip() def build(self, scratch_dir, module_name): args = ['clean', 'install', '-pl', ':' + module_name, '-am'] - ret_code, stdout, stderr = self._run(args, scratch_dir) + ret_code, stdout, _ = self._run(args, scratch_dir) LOG.debug("Maven logs: %s", stdout.decode('utf8').strip()) if ret_code != 0: - raise MavenExecutionError(message=stderr.decode('utf8').strip()) + raise MavenExecutionError(message=stdout.decode('utf8').strip()) def copy_dependency(self, scratch_dir, module_name): args = ['dependency:copy-dependencies', '-DincludeScope=compile', '-pl', ':' + module_name] - ret_code, _, stderr = self._run(args, scratch_dir) + ret_code, stdout, _ = self._run(args, scratch_dir) if ret_code != 0: - raise MavenExecutionError(message=stderr.decode('utf8').strip()) + raise MavenExecutionError(message=stdout.decode('utf8').strip()) def _run(self, args, cwd=None): p = self.os_utils.popen([self.maven_binary.binary_path] + args, cwd=cwd, stdout=subprocess.PIPE, diff --git a/tests/unit/workflows/java_maven/test_maven.py b/tests/unit/workflows/java_maven/test_maven.py index 4ad74576b..ec7c7501e 100644 --- a/tests/unit/workflows/java_maven/test_maven.py +++ b/tests/unit/workflows/java_maven/test_maven.py @@ -52,7 +52,7 @@ def test_retrieve_module_name(self): cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) def test_retrieve_module_name_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, err=b'Some Error Message') + self.popen = FakePopen(retcode=1, out=b'Some Error Message') self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: @@ -67,7 +67,7 @@ def test_build_project(self): cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) def test_build_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, err=b'Some Error Message') + self.popen = FakePopen(retcode=1, out=b'Some Error Message') self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: @@ -82,7 +82,7 @@ def test_copy_dependency(self): cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) def test_copy_dependency_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, err=b'Some Error Message') + self.popen = FakePopen(retcode=1, out=b'Some Error Message') self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: From 95327c3d23fd83a72c20d45d9a42455cdb772f5c Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 13 Mar 2019 11:52:01 -0700 Subject: [PATCH 2/4] fix: Remove MODULE_NAME command from Maven Workflow (#96) * fix: Remove MODULE_NAME command from Maven Workflow On Windows, the echo command is not valid in most cases. The Maven Workflow was doing some operations that help support the multiple project build use-case. Given this sometimes fails on Windows, the PR updates the Maven Worklfow to only install and copy dependencies. This removes the need to echo and therefore allows Windows Maven builds to succeed without needing to run within the container. * remove unused module_name from SubprocessMaven functions --- .../workflows/java_maven/DESIGN.md | 10 ++----- .../workflows/java_maven/actions.py | 17 ++--------- .../workflows/java_maven/maven.py | 17 ++++------- .../unit/workflows/java_maven/test_actions.py | 8 ++---- tests/unit/workflows/java_maven/test_maven.py | 28 ++++--------------- 5 files changed, 18 insertions(+), 62 deletions(-) diff --git a/aws_lambda_builders/workflows/java_maven/DESIGN.md b/aws_lambda_builders/workflows/java_maven/DESIGN.md index bf35b3a3b..1ec7aeaf5 100644 --- a/aws_lambda_builders/workflows/java_maven/DESIGN.md +++ b/aws_lambda_builders/workflows/java_maven/DESIGN.md @@ -71,21 +71,17 @@ configured to use a different one than can be found on the PATH. #### Step 3: Build and package -We leverage Maven to do all the heavy lifting for executing the`mvn package` which +We leverage Maven to do all the heavy lifting for executing the`mvn clean install` which will resolve and download the dependencies and build the project. Built java classes will be located in `target/classes`. Then we use `mvn dependency:copy-dependenceis` to copy the dependencies and the dependencies will be located in `target/dependency` under the source directory. ```bash -MODULE_NAME=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.artifactId}' exec:exec --non-recursive) -mvn clean package -pl :MODULE_NAME -am -mvn dependency:copy-dependencies -DincludeScope=compile -pl :MODULE_NAME +mvn clean install +mvn dependency:copy-dependencies -DincludeScope=compile ``` -Here `MODULE_NAME` is the `artifactId` defined in the pom.xml of the project. Maven -will build the dependencies of that project in the reactor and then build the project itself. - #### Step 4: Copy to artifact directory Built Java classes and dependencies are copied from `scratch_dir/target/classes` and `scratch_dir/target/dependency` diff --git a/aws_lambda_builders/workflows/java_maven/actions.py b/aws_lambda_builders/workflows/java_maven/actions.py index a96fb7d03..e81804d42 100644 --- a/aws_lambda_builders/workflows/java_maven/actions.py +++ b/aws_lambda_builders/workflows/java_maven/actions.py @@ -20,17 +20,6 @@ def __init__(self, subprocess_maven): self.scratch_dir = scratch_dir self.subprocess_maven = subprocess_maven - self.artifact_id = None - - @property - def module_name(self): - if self.artifact_id is None: - try: - self.artifact_id = self.subprocess_maven.retrieve_module_name(self.scratch_dir) - except MavenExecutionError as ex: - raise ActionFailedError(str(ex)) - - return self.artifact_id class JavaMavenBuildAction(JavaMavenBaseAction, BaseAction): @@ -48,8 +37,7 @@ def __init__(self, def execute(self): try: - self.subprocess_maven.build(self.scratch_dir, - self.module_name) + self.subprocess_maven.build(self.scratch_dir) except MavenExecutionError as ex: raise ActionFailedError(str(ex)) @@ -69,8 +57,7 @@ def __init__(self, def execute(self): try: - self.subprocess_maven.copy_dependency(self.scratch_dir, - self.module_name) + self.subprocess_maven.copy_dependency(self.scratch_dir) except MavenExecutionError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/java_maven/maven.py b/aws_lambda_builders/workflows/java_maven/maven.py index dd778f792..aa6f55ad0 100644 --- a/aws_lambda_builders/workflows/java_maven/maven.py +++ b/aws_lambda_builders/workflows/java_maven/maven.py @@ -25,16 +25,8 @@ def __init__(self, maven_binary, os_utils=None): raise ValueError("Must provide OSUtils") self.os_utils = os_utils - def retrieve_module_name(self, scratch_dir): - args = ['-q', '-Dexec.executable=echo', '-Dexec.args=${project.artifactId}', - 'exec:exec', '--non-recursive'] - ret_code, stdout, _ = self._run(args, scratch_dir) - if ret_code != 0: - raise MavenExecutionError(message=stdout.decode('utf8').strip()) - return stdout.decode('utf8').strip() - - def build(self, scratch_dir, module_name): - args = ['clean', 'install', '-pl', ':' + module_name, '-am'] + def build(self, scratch_dir): + args = ['clean', 'install'] ret_code, stdout, _ = self._run(args, scratch_dir) LOG.debug("Maven logs: %s", stdout.decode('utf8').strip()) @@ -42,9 +34,10 @@ def build(self, scratch_dir, module_name): if ret_code != 0: raise MavenExecutionError(message=stdout.decode('utf8').strip()) - def copy_dependency(self, scratch_dir, module_name): - args = ['dependency:copy-dependencies', '-DincludeScope=compile', '-pl', ':' + module_name] + def copy_dependency(self, scratch_dir): + args = ['dependency:copy-dependencies', '-DincludeScope=compile'] ret_code, stdout, _ = self._run(args, scratch_dir) + if ret_code != 0: raise MavenExecutionError(message=stdout.decode('utf8').strip()) diff --git a/tests/unit/workflows/java_maven/test_actions.py b/tests/unit/workflows/java_maven/test_actions.py index a2676eb25..6e4986c88 100644 --- a/tests/unit/workflows/java_maven/test_actions.py +++ b/tests/unit/workflows/java_maven/test_actions.py @@ -14,14 +14,12 @@ def setUp(self, MockSubprocessMaven): self.subprocess_maven = MockSubprocessMaven.return_value self.scratch_dir = os.path.join('scratch_dir') self.artifacts_dir = os.path.join('artifacts_dir') - self.module_name = "module" def test_calls_maven_build(self): - self.subprocess_maven.retrieve_module_name.side_effect = lambda scratch: self.module_name action = JavaMavenBuildAction(self.scratch_dir, self.subprocess_maven) action.execute() - self.subprocess_maven.build.assert_called_with(self.scratch_dir, self.module_name) + self.subprocess_maven.build.assert_called_with(self.scratch_dir) def test_error_building_project_raises_action_error(self): self.subprocess_maven.build.side_effect = MavenExecutionError(message='Build failed!') @@ -39,14 +37,12 @@ def setUp(self, MockSubprocessMaven): self.subprocess_maven = MockSubprocessMaven.return_value self.scratch_dir = os.path.join('scratch_dir') self.artifacts_dir = os.path.join('artifacts_dir') - self.module_name = 'module_name' def test_calls_maven_copy_dependency(self): - self.subprocess_maven.retrieve_module_name.side_effect = lambda scratch: self.module_name action = JavaMavenCopyDependencyAction(self.scratch_dir, self.subprocess_maven) action.execute() - self.subprocess_maven.copy_dependency.assert_called_with(self.scratch_dir, self.module_name) + self.subprocess_maven.copy_dependency.assert_called_with(self.scratch_dir) def test_error_building_project_raises_action_error(self): self.subprocess_maven.copy_dependency.side_effect = MavenExecutionError(message='Build failed!') diff --git a/tests/unit/workflows/java_maven/test_maven.py b/tests/unit/workflows/java_maven/test_maven.py index ec7c7501e..cd2181234 100644 --- a/tests/unit/workflows/java_maven/test_maven.py +++ b/tests/unit/workflows/java_maven/test_maven.py @@ -43,27 +43,11 @@ def test_no_maven_exec_init_throws(self): SubprocessMaven(None) self.assertEquals(err_assert.exception.args[0], 'Must provide Maven BinaryPath') - def test_retrieve_module_name(self): - maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) - maven.retrieve_module_name(self.source_dir) - self.os_utils.popen.assert_called_with( - [self.maven_path, '-q', '-Dexec.executable=echo', '-Dexec.args=${project.artifactId}', - 'exec:exec', '--non-recursive'], - cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - - def test_retrieve_module_name_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, out=b'Some Error Message') - self.os_utils.popen.side_effect = [self.popen] - maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) - with self.assertRaises(MavenExecutionError) as err: - maven.retrieve_module_name(self.source_dir) - self.assertEquals(err.exception.args[0], 'Maven Failed: Some Error Message') - def test_build_project(self): maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) - maven.build(self.source_dir, self.module_name) + maven.build(self.source_dir) self.os_utils.popen.assert_called_with( - [self.maven_path, 'clean', 'install', '-pl', ':' + self.module_name, '-am'], + [self.maven_path, 'clean', 'install'], cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) def test_build_raises_exception_if_retcode_not_0(self): @@ -71,14 +55,14 @@ def test_build_raises_exception_if_retcode_not_0(self): self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: - maven.build(self.source_dir, self.module_name) + maven.build(self.source_dir) self.assertEquals(err.exception.args[0], 'Maven Failed: Some Error Message') def test_copy_dependency(self): maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) - maven.copy_dependency(self.source_dir, self.module_name) + maven.copy_dependency(self.source_dir) self.os_utils.popen.assert_called_with( - [self.maven_path, 'dependency:copy-dependencies', '-DincludeScope=compile', '-pl', ':' + self.module_name], + [self.maven_path, 'dependency:copy-dependencies', '-DincludeScope=compile'], cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) def test_copy_dependency_raises_exception_if_retcode_not_0(self): @@ -86,5 +70,5 @@ def test_copy_dependency_raises_exception_if_retcode_not_0(self): self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: - maven.copy_dependency(self.source_dir, self.module_name) + maven.copy_dependency(self.source_dir) self.assertEquals(err.exception.args[0], 'Maven Failed: Some Error Message') From e335171a51549aca16198ae412642aadfa8bb434 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+TheSriram@users.noreply.github.com> Date: Wed, 13 Mar 2019 11:52:09 -0700 Subject: [PATCH 3/4] fix: go validator (#93) * fix: go validator * fix: remove pin on golang --- .appveyor.yml | 2 +- aws_lambda_builders/workflows/go_modules/validator.py | 2 +- tests/unit/workflows/go_modules/test_validator.py | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index e2a78b322..3c8635bb3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -30,7 +30,7 @@ install: # setup go - rmdir c:\go /s /q -- "choco install golang --version 1.11.5" +- "choco install golang" - "choco install bzr" - "choco install dep" - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;" diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index e44779efc..872701137 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -53,7 +53,7 @@ def validate(self, runtime_path): out_parts = out.decode().split() if len(out_parts) >= 3: version_parts = [int(x) for x in out_parts[2].replace(self.LANGUAGE, "").split('.')] - if len(version_parts) == 3: + if len(version_parts) >= 2: if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version: self._valid_runtime_path = runtime_path return self._valid_runtime_path diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index 9e3931de7..40486c8f7 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -40,6 +40,12 @@ def test_runtime_validate_supported_version_runtime(self): self.validator.validate(runtime_path="/usr/bin/go") self.assertTrue(mock_subprocess.call_count, 1) + def test_runtime_validate_supported_higher_than_min_version_runtime(self): + with mock.patch("subprocess.Popen") as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.12 test") + self.validator.validate(runtime_path="/usr/bin/go") + self.assertTrue(mock_subprocess.call_count, 1) + def test_runtime_validate_mismatch_nonzero_exit(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) From 8729f21156474065ab426f389451fc623c1907a0 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 19 Mar 2019 12:38:51 -0700 Subject: [PATCH 4/4] chore: Release v0.2.1 (#102) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 5ffd033a3..20063ad1b 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = '0.2.0' +__version__ = '0.2.1' RPC_PROTOCOL_VERSION = "0.2"