Merge branch 'develop'

aws_lambda_builders/__init__.py

@@ -4,5 +4,5 @@
 
 # Changing version will trigger a new release!
 # Please make the version change as the last step of your development.
-__version__ = "1.38.0"
+__version__ = "1.39.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/utils.py

@@ -1,7 +1,7 @@
 """
 Common utilities for the library
 """
-
+import locale
 import logging
 import os
 import shutil
@@ -231,3 +231,27 @@ def extract_tarfile(tarfile_path: Union[str, os.PathLike], unpack_dir: Union[str
                 raise tarfile.ExtractError("Attempted Path Traversal in Tar File")
 
         tar.extractall(unpack_dir)
+
+
+def decode(to_decode: bytes, encoding: Optional[str] = None) -> str:
+    """
+    Perform a "safe" decoding of a series of bytes. Attempts to find the localized encoding
+    if not provided, and avoids raising an exception, instead, if an unrecognized character
+    is found, replaces it with a replacement character.
+
+    https://docs.python.org/3/library/codecs.html#codec-base-classes
+
+    Parameters
+    ----------
+    to_decode: bytes
+        Series of bytes to be decoded
+    encoding: Optional[str]
+        Encoding type. If None, will attempt to find the correct encoding based on locale.
+
+    Returns
+    -------
+    str
+       Decoded string with unrecognized characters replaced with a replacement character
+    """
+    encoding = encoding or locale.getpreferredencoding()
+    return to_decode.decode(encoding, errors="replace").strip()

aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py

@@ -2,10 +2,10 @@
 Wrapper around calls to dotent CLI through a subprocess.
 """
 
-import locale
 import logging
 
-from .utils import OSUtils
+from aws_lambda_builders.utils import decode
+from aws_lambda_builders.workflows.dotnet_clipackage.utils import OSUtils
 
 LOG = logging.getLogger(__name__)
 
@@ -52,15 +52,14 @@ def run(self, args, cwd=None):
         # DotNet output is in system locale dependent encoding
         # https://learn.microsoft.com/en-us/dotnet/api/system.console.outputencoding?view=net-6.0#remarks
         # "The default code page that the console uses is determined by the system locale."
-        encoding = locale.getpreferredencoding()
         p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd)
 
         out, err = p.communicate()
 
         # The package command contains lots of useful information on how the package was created and
         # information when the package command was not successful. For that reason the output is
         # always written to the output to help developers diagnose issues.
-        LOG.info(out.decode(encoding).strip())
+        LOG.info(decode(out))
 
         if p.returncode != 0:
-            raise DotnetCLIExecutionError(message=err.decode(encoding).strip())
+            raise DotnetCLIExecutionError(message=decode(err))

aws_lambda_builders/workflows/dotnet_clipackage/utils.py

@@ -7,7 +7,7 @@
 import subprocess
 import zipfile
 
-from aws_lambda_builders.utils import which
+from aws_lambda_builders.utils import decode, which
 
 LOG = logging.getLogger(__name__)
 
@@ -96,7 +96,7 @@ def _extract(self, file_info, output_dir, zip_ref):
         if not self._is_symlink(file_info):
             return zip_ref.extract(file_info, output_dir)
 
-        source = zip_ref.read(file_info.filename).decode("utf8")
+        source = decode(zip_ref.read(file_info.filename))
         link_name = os.path.normpath(os.path.join(output_dir, file_info.filename))
 
         # make leading dirs if needed

aws_lambda_builders/workflows/java_gradle/gradle_validator.py

@@ -5,6 +5,7 @@
 import logging
 import re
 
+from aws_lambda_builders.utils import decode
 from aws_lambda_builders.validator import RuntimeValidator
 from aws_lambda_builders.workflows.java.utils import OSUtils
 
@@ -81,6 +82,6 @@ def _get_jvm_string(self, gradle_path):
             return None
 
         for line in stdout.splitlines():
-            l_dec = line.decode()
+            l_dec = decode(line)
             if l_dec.startswith("JVM"):
                 return l_dec

aws_lambda_builders/workflows/java_maven/maven.py

@@ -5,6 +5,8 @@
 import logging
 import subprocess
 
+from aws_lambda_builders.utils import decode
+
 LOG = logging.getLogger(__name__)
 
 
@@ -28,10 +30,10 @@ def build(self, scratch_dir):
         args = ["clean", "install"]
         ret_code, stdout, _ = self._run(args, scratch_dir)
 
-        LOG.debug("Maven logs: %s", stdout.decode("utf8").strip())
+        LOG.debug("Maven logs: %s", decode(stdout))
 
         if ret_code != 0:
-            raise MavenExecutionError(message=stdout.decode("utf8").strip())
+            raise MavenExecutionError(message=decode(stdout))
 
     def copy_dependency(self, scratch_dir):
         include_scope = "runtime"
@@ -40,7 +42,7 @@ def copy_dependency(self, scratch_dir):
         ret_code, stdout, _ = self._run(args, scratch_dir)
 
         if ret_code != 0:
-            raise MavenExecutionError(message=stdout.decode("utf8").strip())
+            raise MavenExecutionError(message=decode(stdout))
 
     def _run(self, args, cwd=None):
         p = self.os_utils.popen(

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -7,8 +7,7 @@
 
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from aws_lambda_builders.utils import extract_tarfile
-
-from .npm import NpmExecutionError, SubprocessNpm
+from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError, SubprocessNpm
 
 LOG = logging.getLogger(__name__)
 

aws_lambda_builders/workflows/nodejs_npm/exceptions.py

@@ -0,0 +1,18 @@
+"""
+Exceptions for the Node.js workflow
+"""
+
+
+from aws_lambda_builders.exceptions import LambdaBuilderError
+
+
+class NpmExecutionError(LambdaBuilderError):
+    """
+    Exception raised in case NPM execution fails.
+    It will pass on the standard error output from the NPM console.
+    """
+
+    MESSAGE = "NPM Failed: {message}"
+
+    def __init__(self, **kwargs):
+        Exception.__init__(self, self.MESSAGE.format(**kwargs))

aws_lambda_builders/workflows/nodejs_npm/npm.py

@@ -4,20 +4,9 @@
 
 import logging
 
-LOG = logging.getLogger(__name__)
-
-
-class NpmExecutionError(Exception):
-
-    """
-    Exception raised in case NPM execution fails.
-    It will pass on the standard error output from the NPM console.
-    """
+from aws_lambda_builders.workflows.nodejs_npm.exceptions import NpmExecutionError
 
-    MESSAGE = "NPM Failed: {message}"
-
-    def __init__(self, **kwargs):
-        Exception.__init__(self, self.MESSAGE.format(**kwargs))
+LOG = logging.getLogger(__name__)
 
 
 class SubprocessNpm(object):
@@ -59,7 +48,7 @@ def run(self, args, cwd=None):
         :rtype: str
         :return: text of the standard output from the command
 
-        :raises aws_lambda_builders.workflows.nodejs_npm.npm.NpmExecutionError:
+        :raises aws_lambda_builders.workflows.nodejs_npm.exceptions.NpmExecutionError:
             when the command executes with a non-zero return code. The exception will
             contain the text of the standard error output from the command.
 

aws_lambda_builders/workflows/nodejs_npm/workflow.py

@@ -15,20 +15,30 @@
 )
 from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability
-
-from .actions import (
+from aws_lambda_builders.workflows.nodejs_npm.actions import (
     NodejsNpmCIAction,
     NodejsNpmInstallAction,
     NodejsNpmLockFileCleanUpAction,
     NodejsNpmPackAction,
     NodejsNpmrcAndLockfileCopyAction,
     NodejsNpmrcCleanUpAction,
 )
-from .npm import SubprocessNpm
-from .utils import OSUtils
+from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm
+from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils
 
 LOG = logging.getLogger(__name__)
 
+# npm>=8.8.0 supports --install-links
+MINIMUM_NPM_VERSION_INSTALL_LINKS = (8, 8)
+UNSUPPORTED_NPM_VERSION_MESSAGE = (
+    "Building in source was enabled, however the "
+    "currently installed npm version does not support "
+    "--install-links. Please ensure that the npm "
+    "version is at least 8.8.0. Switching to build "
+    f"in outside of the source directory.{os.linesep}"
+    "https://docs.npmjs.com/cli/v8/using-npm/changelog#v880-2022-04-27"
+)
+
 
 class NodejsNpmWorkflow(BaseWorkflow):
 
@@ -89,6 +99,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
             self.actions.append(CopySourceAction(self.source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
 
         if self.download_dependencies:
+            if is_building_in_source and not self.can_use_install_links(subprocess_npm):
+                LOG.warning(UNSUPPORTED_NPM_VERSION_MESSAGE)
+
+                is_building_in_source = False
+                self.build_dir = self._select_build_dir(build_in_source=False)
+
             self.actions.append(
                 NodejsNpmWorkflow.get_install_action(
                     source_dir=source_dir,
@@ -235,3 +251,40 @@ def get_install_action(
         return NodejsNpmInstallAction(
             install_dir=install_dir, subprocess_npm=subprocess_npm, install_links=install_links
         )
+
+    @staticmethod
+    def can_use_install_links(npm_process: SubprocessNpm) -> bool:
+        """
+        Checks the version of npm that is currently installed to determine
+        whether or not --install-links can be used
+
+        Parameters
+        ----------
+        npm_process: SubprocessNpm
+            Object containing helper methods to call the npm process
+
+        Returns
+        -------
+        bool
+            True if the current npm version meets the minimum for --install-links
+        """
+        try:
+            current_version = npm_process.run(["--version"])
+
+            LOG.debug(f"Currently installed version of npm is: {current_version}")
+
+            current_version = current_version.split(".")
+
+            major_version = int(current_version[0])
+            minor_version = int(current_version[1])
+        except (ValueError, IndexError):
+            LOG.debug(f"Failed to parse {current_version} output from npm for --install-links validation")
+            return False
+
+        is_older_major_version = major_version < MINIMUM_NPM_VERSION_INSTALL_LINKS[0]
+        is_older_patch_version = (
+            major_version == MINIMUM_NPM_VERSION_INSTALL_LINKS[0]
+            and minor_version < MINIMUM_NPM_VERSION_INSTALL_LINKS[1]
+        )
+
+        return not (is_older_major_version or is_older_patch_version)

aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py

@@ -14,17 +14,17 @@
     LinkSourceAction,
     MoveDependenciesAction,
 )
+from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.utils import which
 from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability
-
-from ...path_resolver import PathResolver
-from ..nodejs_npm import NodejsNpmWorkflow
-from ..nodejs_npm.npm import SubprocessNpm
-from ..nodejs_npm.utils import OSUtils
-from .actions import (
+from aws_lambda_builders.workflows.nodejs_npm import NodejsNpmWorkflow
+from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm
+from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils
+from aws_lambda_builders.workflows.nodejs_npm.workflow import UNSUPPORTED_NPM_VERSION_MESSAGE
+from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import (
     EsbuildBundleAction,
 )
-from .esbuild import EsbuildExecutionError, SubprocessEsbuild
+from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import EsbuildExecutionError, SubprocessEsbuild
 
 LOG = logging.getLogger(__name__)
 
@@ -98,6 +98,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
             )
 
         if self.download_dependencies:
+            if is_building_in_source and not NodejsNpmWorkflow.can_use_install_links(self.subprocess_npm):
+                LOG.warning(UNSUPPORTED_NPM_VERSION_MESSAGE)
+
+                is_building_in_source = False
+                self.build_dir = self._select_build_dir(build_in_source=False)
+
             self.actions.append(
                 NodejsNpmWorkflow.get_install_action(
                     source_dir=source_dir,
@@ -177,8 +183,16 @@ def get_resolvers(self):
         return [PathResolver(runtime=self.runtime, binary="npm")]
 
     def _get_esbuild_subprocess(self) -> SubprocessEsbuild:
+        """
+        Creates a subprocess object that is able to invoke the esbuild executable.
+
+        Returns
+        -------
+        SubprocessEsbuild
+            An esbuild specific subprocess object
+        """
         try:
-            npm_bin_path_root = self.subprocess_npm.run(["root"], cwd=self.scratch_dir)
+            npm_bin_path_root = self.subprocess_npm.run(["root"], cwd=self.build_dir)
             npm_bin_path = str(Path(npm_bin_path_root, ".bin"))
         except FileNotFoundError:
             raise EsbuildExecutionError(message="The esbuild workflow couldn't find npm installed on your system.")