Merge pull request #306 from aws/develop

chore: Lambda Builder 1.9.0 Release Merge to Master

.appveyor.yml

@@ -9,92 +9,81 @@ environment:
   nodejs_version: "10.10.0"
 
   matrix:
-
-  - PYTHON: "C:\\Python27-x64"
-    PYTHON_VERSION: '2.7'
-    PYTHON_ARCH: '64'
-    LINE_COVERAGE: '91'
-    NEW_FLAKE8: 0
-    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
-  - PYTHON: "C:\\Python36-x64"
-    PYTHON_VERSION: '3.6'
-    PYTHON_ARCH: '64'
-    LINE_COVERAGE: '91'
-    NEW_FLAKE8: 0
-    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
-  - PYTHON: "C:\\Python37-x64"
-    PYTHON_VERSION: '3.7'
-    PYTHON_ARCH: '64'
-    LINE_COVERAGE: '91'
-    NEW_FLAKE8: 0
-    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
-  - PYTHON: "C:\\Python38-x64"
-    PYTHON_VERSION: '3.8'
-    PYTHON_ARCH: '64'
-    LINE_COVERAGE: '72'
-    NEW_FLAKE8: 1
-    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
-  - PYTHON: "C:\\Python39-x64"
-    PYTHON_VERSION: '3.9'
-    PYTHON_ARCH: '64'
-    LINE_COVERAGE: '72'
-    NEW_FLAKE8: 1
-    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
-
-
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6"
+      PYTHON_ARCH: "64"
+      LINE_COVERAGE: "91"
+      NEW_FLAKE8: 0
+      JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7"
+      PYTHON_ARCH: "64"
+      LINE_COVERAGE: "91"
+      NEW_FLAKE8: 0
+      JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
+    - PYTHON: "C:\\Python38-x64"
+      PYTHON_VERSION: "3.8"
+      PYTHON_ARCH: "64"
+      LINE_COVERAGE: "72"
+      NEW_FLAKE8: 1
+      JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
+    - PYTHON: "C:\\Python39-x64"
+      PYTHON_VERSION: "3.9"
+      PYTHON_ARCH: "64"
+      LINE_COVERAGE: "72"
+      NEW_FLAKE8: 1
+      JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
 
 build: off
 
 for:
-  -
-    matrix:
+  - matrix:
       only:
         - image: Visual Studio 2019
 
     environment:
       GOPATH: c:\gopath
 
     install:
-    # To run Nodejs workflow integ tests
-    - ps: Install-Product node $env:nodejs_version
-
-    - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
-    - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
-    - "%PYTHON%\\python.exe -m pip install -e ."
-    - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
-    - "gem --version"
-    - "gem install bundler -v 1.17.3"
-    - "bundler --version"
-    - "echo %PATH%"
-
-    # setup go
-    - rmdir c:\go /s /q
-    - "choco install golang --version 1.15.7"
-    - "choco install bzr"
-    - "choco install dep"
-    - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
-    - "go version"
-    # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
-    # https://blog.golang.org/go116-module-changes#TOC_2.
-    # This is required for the go dep integration tests
-    - "go env -w GO111MODULE=auto"
-    - "go env"
-
-    # setup Gradle
-    - "choco install gradle"
-
-    # setup make
-    - "choco install make"
-
-    # Echo final Path
-    - "echo %PATH%"
+      # To run Nodejs workflow integ tests
+      - ps: Install-Product node $env:nodejs_version
+
+      - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+      - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
+      - "%PYTHON%\\python.exe -m pip install -e ."
+      - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
+      - "gem --version"
+      - "gem install bundler -v 1.17.3"
+      - "bundler --version"
+      - "echo %PATH%"
+
+      # setup go
+      - rmdir c:\go /s /q
+      - "choco install golang --version 1.15.7"
+      - "choco install bzr"
+      - "choco install dep"
+      - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
+      - "go version"
+      # set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
+      # https://blog.golang.org/go116-module-changes#TOC_2.
+      # This is required for the go dep integration tests
+      - "go env -w GO111MODULE=auto"
+      - "go env"
+
+      # setup Gradle
+      - "choco install gradle"
+
+      # setup make
+      - "choco install make"
+
+      # Echo final Path
+      - "echo %PATH%"
 
     test_script:
-    - "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
-    - "%PYTHON%\\python.exe -m pytest tests/integration"
+      - "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
+      - "%PYTHON%\\python.exe -m pytest tests/integration"
 
-  -
-    matrix:
+  - matrix:
       only:
         - image: Ubuntu
 
@@ -121,8 +110,8 @@ for:
       - sh: "go env -w GO111MODULE=auto"
 
     build_script:
-      - "python -c \"import sys; print(sys.executable)\""
-      - "LAMBDA_BUILDERS_DEV=1 pip install -e \".[dev]\""
+      - 'python -c "import sys; print(sys.executable)"'
+      - 'LAMBDA_BUILDERS_DEV=1 pip install -e ".[dev]"'
 
     test_script:
       - "LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under ${LINE_COVERAGE} tests/unit tests/functional"

Makefile

@@ -28,6 +28,3 @@ black-check:
 
 # Verifications to run before sending a pull request
 pr: init dev black-check
-
-# Verifications to run before sending a pull request, skipping black check because black requires Python 3.6+
-pr2.7: init dev

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.8.1"
+__version__ = "1.9.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/__main__.py

@@ -125,6 +125,9 @@ def main():  # pylint: disable=too-many-statements
             optimizations=params["optimizations"],
             options=params["options"],
             mode=params.get("mode", None),
+            download_dependencies=params.get("download_dependencies", True),
+            dependencies_dir=params.get("dependencies_dir", None),
+            combine_dependencies=params.get("combine_dependencies", True),
             architecture=params.get("architecture", X86_64),
         )
 

aws_lambda_builders/actions.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+import os
 import shutil
 import six
 
@@ -30,9 +31,18 @@ class Purpose(object):
     # Action is copying source code
     COPY_SOURCE = "COPY_SOURCE"
 
+    # Action is copying dependencies
+    COPY_DEPENDENCIES = "COPY_DEPENDENCIES"
+
+    # Action is moving dependencies
+    MOVE_DEPENDENCIES = "MOVE_DEPENDENCIES"
+
     # Action is compiling source code
     COMPILE_SOURCE = "COMPILE_SOURCE"
 
+    # Action is cleaning up the target folder
+    CLEAN_UP = "CLEAN_UP"
+
     @staticmethod
     def has_value(item):
         return item in Purpose.__dict__.values()
@@ -99,3 +109,87 @@ def __init__(self, source_dir, dest_dir, excludes=None):
 
     def execute(self):
         copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))
+
+
+class CopyDependenciesAction(BaseAction):
+
+    NAME = "CopyDependencies"
+
+    DESCRIPTION = "Copying dependencies while skipping source file"
+
+    PURPOSE = Purpose.COPY_DEPENDENCIES
+
+    def __init__(self, source_dir, artifact_dir, destination_dir):
+        self.source_dir = source_dir
+        self.artifact_dir = artifact_dir
+        self.dest_dir = destination_dir
+
+    def execute(self):
+        source = set(os.listdir(self.source_dir))
+        artifact = set(os.listdir(self.artifact_dir))
+        dependencies = artifact - source
+
+        for name in dependencies:
+            dependencies_source = os.path.join(self.artifact_dir, name)
+            new_destination = os.path.join(self.dest_dir, name)
+
+            if os.path.isdir(dependencies_source):
+                copytree(dependencies_source, new_destination)
+            else:
+                shutil.copy2(dependencies_source, new_destination)
+
+
+class MoveDependenciesAction(BaseAction):
+
+    NAME = "MoveDependencies"
+
+    DESCRIPTION = "Moving dependencies while skipping source file"
+
+    PURPOSE = Purpose.MOVE_DEPENDENCIES
+
+    def __init__(self, source_dir, artifact_dir, destination_dir):
+        self.source_dir = source_dir
+        self.artifact_dir = artifact_dir
+        self.dest_dir = destination_dir
+
+    def execute(self):
+        source = set(os.listdir(self.source_dir))
+        artifact = set(os.listdir(self.artifact_dir))
+        dependencies = artifact - source
+
+        for name in dependencies:
+            dependencies_source = os.path.join(self.artifact_dir, name)
+            new_destination = os.path.join(self.dest_dir, name)
+
+            shutil.move(dependencies_source, new_destination)
+
+
+class CleanUpAction(BaseAction):
+    """
+    Class for cleaning the directory. It will clean all the files in the directory but doesn't delete the directory
+    """
+
+    NAME = "CleanUp"
+
+    DESCRIPTION = "Cleaning up the target folder"
+
+    PURPOSE = Purpose.CLEAN_UP
+
+    def __init__(self, target_dir):
+        self.target_dir = target_dir
+
+    def execute(self):
+        if not os.path.isdir(self.target_dir):
+            LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
+            return
+        targets = os.listdir(self.target_dir)
+        LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir))
+
+        for name in targets:
+            target_path = os.path.join(self.target_dir, name)
+            LOG.debug("Clean up action: %s is deleted", str(target_path))
+
+            if os.path.isdir(target_path):
+                shutil.rmtree(target_path)
+            else:
+                os.remove(target_path)

aws_lambda_builders/builder.py

@@ -65,6 +65,9 @@ def build(
         options=None,
         executable_search_paths=None,
         mode=None,
+        download_dependencies=True,
+        dependencies_dir=None,
+        combine_dependencies=True,
         architecture=X86_64,
     ):
         """
@@ -108,6 +111,19 @@ def build(
         :param mode:
             Optional, Mode the build should produce
 
+        :type download_dependencies: bool
+        :param download_dependencies:
+            Optional, Should download dependencies when building
+
+        :type dependencies_dir: str
+        :param dependencies_dir:
+            Optional, Path to folder the dependencies should be downloaded to
+
+        :type combine_dependencies: bool
+        :param combine_dependencies:
+            Optional, This flag will only be used if dependency_folder is specified. False will not copy dependencies
+            from dependency_folder into build folder
+
         :type architecture: str
         :param architecture:
             Type of architecture x86_64 and arm64 for Lambda Function
@@ -126,6 +142,9 @@ def build(
             options=options,
             executable_search_paths=executable_search_paths,
             mode=mode,
+            download_dependencies=download_dependencies,
+            dependencies_dir=dependencies_dir,
+            combine_dependencies=combine_dependencies,
             architecture=architecture,
         )
 

aws_lambda_builders/utils.py

@@ -31,6 +31,10 @@ def copytree(source, destination, ignore=None):
         ``ignore`` property of ``shutils.copytree`` method
     """
 
+    if not os.path.exists(source):
+        LOG.warning("Skipping copy operation since source %s does not exist", source)
+        return
+
     if not os.path.exists(destination):
         os.makedirs(destination)
 

aws_lambda_builders/workflow.py

@@ -160,6 +160,9 @@ def __init__(
         optimizations=None,
         options=None,
         mode=BuildMode.RELEASE,
+        download_dependencies=True,
+        dependencies_dir=None,
+        combine_dependencies=True,
         architecture=X86_64,
     ):
         """
@@ -178,7 +181,7 @@ def __init__(
         manifest_path : str
             Path to the dependency manifest
         runtime : str, optional
-            name of the AWS Lambda runtime that you are building for. This is sent to the builder for
+            Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for
             informational purposes, by default None
         executable_search_paths : list, optional
             Additional list of paths to search for executables required by the workflow, by default None
@@ -188,6 +191,13 @@ def __init__(
             dictionary of options ot pass to build action. **Not supported**., by default None
         mode : str, optional
             Mode the build should produce, by default BuildMode.RELEASE
+        download_dependencies: bool, optional
+            Should download dependencies when building
+        dependencies_dir : str, optional
+            Path to folder the dependencies should be downloaded to
+        combine_dependencies: bool, optional
+            This flag will only be used if dependency_folder is specified. False will not copy dependencies
+            from dependency_folder into build folder
         architecture : str, optional
             Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64
         """
@@ -201,6 +211,9 @@ def __init__(
         self.options = options
         self.executable_search_paths = executable_search_paths
         self.mode = mode
+        self.download_dependencies = download_dependencies
+        self.dependencies_dir = dependencies_dir
+        self.combine_dependencies = combine_dependencies
         self.architecture = architecture
 
         # Actions are registered by the subclasses as they seem fit