Release v0.4.0

CHANGELOG.md

@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - 2021-06-11
+
+### Added
+
+- Now possible to execute services in stages (#227)
+- Service policies (#236)
+- Supply private key to ansible using `stackl_private_key` (#248)
+
+### Changed
+
+- Ansible outputs now make use of callback plugin (#242)
+- New way of supplying command args for all the handlers
+- Terraform backend can now be a jinja template (#265)
+- Vault secrets also can be used as env variables with envconsul 
+
+### Fixed
+
+- Generic invocation now gets chosen by default
+
 ## [0.3.3] - 2021-02-18
 
 ### Fixed
@@ -93,7 +112,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add reporting policies
 - Add codeql-analysis
 
-[unreleased]: https://github.com/stacklio/stackl/compare/v0.3.3...HEAD
+[unreleased]: https://github.com/stacklio/stackl/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/stacklio/stackl/compare/v0.3.3...v0.4.0
 [0.3.3]: https://github.com/stacklio/stackl/compare/v0.3.2...v0.3.3
 [0.3.2]: https://github.com/stacklio/stackl/compare/v0.3.1...v0.3.2
 [0.3.1]: https://github.com/stacklio/stackl/compare/v0.3.0...v0.3.1

stackl/agent/agent/kubernetes/handlers/ansible_handler.py

@@ -14,19 +14,16 @@
 PLAYBOOK_INCLUDE_ROLE = """
 - hosts: "{{ pattern }}"
   serial: "{{ serial }}"
-  gather_facts: no
+  connection: "{{ connection }}"
+  gather_facts: "{{ stackl_gather_facts }}"
+  pre_tasks:
+    - wait_for:     
+        port: "{{ wait_for_port | int }}"     
+        timeout: 60
+      when: wait_for_port is defined
   tasks:
     - include_role:
         name: "{{ ansible_role }}"
-- hosts: localhost
-  connection: local
-  gather_facts: no
-  tasks:
-    - set_fact:
-        output_dict: "{{ hostvars }}"
-    - copy:
-        content: "{{ output_dict | to_nice_json }}"
-        dest: "{{ outputs_path | default('/tmp/outputs.json') }}"
 """
 
 
@@ -122,12 +119,24 @@ def create_command_args(self) -> List[str]:
             else:
                 pattern = self._service + "_" + str(self.index)
 
+            if self._invoc.ansible_role:
+                ansible_role = self._invoc.ansible_role
+            else:
+                ansible_role = self._functional_requirement
+
             ansible_commands = [
-                'ansible', pattern, '-m', 'include_role', '-v', '-i',
-                '/opt/ansible/playbooks/inventory/stackl.yml', '-a',
-                f'name={self._functional_requirement}'
+                'ansible-playbook', '/opt/ansible/playbooks/stackl/playbook-role.yml', '-i',
+                '/opt/ansible/playbooks/inventory/stackl.yml', '-e', f'pattern={pattern}', '-e', f'ansible_role={ansible_role}',
+                '-e', f'serial={self._invoc.serial}', '-e', f'stackl_gather_facts={self._invoc.gather_facts}'
             ]
 
+            if self._invoc.wait_for_port:
+                ansible_commands.extend(['-e', f'wait_for_port={self._invoc.wait_for_port}'])
+
+            if self._invoc.connection:
+                ansible_commands.extend(['-e', f'connection={self._invoc.connection}'])
+
+            ansible_commands.extend(['-e', f'state=present'])
         return ansible_commands
 
     @property

stackl/agent/agent/kubernetes/handlers/packer_handler.py

@@ -74,11 +74,12 @@ def create_command_args(self) -> List[str]:
         if self._secret_handler.secret_variables_file:
             packer_commands += f' -var-file {self._secret_handler.secret_variables_file}'
 
-        packer_commands = self._secret_handler.customize_commands(packer_commands)
         if self._output:
             packer_commands = self._output.customize_commands(
                 packer_commands)
 
+        packer_commands = self._secret_handler.customize_commands(packer_commands)
+
         packer_commands += ' /opt/packer/src/packer.json'
 
         command_args.append(packer_commands)

stackl/agent/agent/kubernetes/handlers/terraform_handler.py

@@ -101,12 +101,14 @@ def create_command_args(self) -> List[str]:
         if self._secret_handler.secret_variables_file:
             terraform_commands += f' -var-file {self._secret_handler.secret_variables_file}'
 
-        terraform_commands = self._secret_handler.customize_commands(
-            terraform_commands)
-
         if self._output:
             terraform_commands = self._output.customize_commands(
                 terraform_commands)
+
+        terraform_commands = self._secret_handler.customize_commands(
+            terraform_commands)
+
+
 
         command_args.append(terraform_commands)
         return command_args

stackl/agent/agent/main.py

@@ -35,6 +35,10 @@ class Invocation:
     tool: str
     hosts: List
     playbook_path: str = None
+    ansible_role: str = None
+    gather_facts: bool = True
+    connection: str = "remote"
+    wait_for_port: int = None
     serial: int = 10
 
 

stackl/agent/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agent"
-version = "0.3.4dev"
+version = "0.4.0"
 description = ""
 authors = ["Frederic <[email protected]>"]
 

stackl/cli/setup.py

@@ -1,14 +1,14 @@
 from setuptools import setup, find_packages
 
-__version__ = "0.3.4dev"
+__version__ = "0.4.0"
 
 setup(
     name='stackl-cli',
     version=__version__,
     py_modules=['stackl', 'commands', 'context'],
     packages=find_packages(),
     install_requires=[
-        f'stackl-client==0.3.4dev', 'pyYAML==5.4.1', 'Click==7.0',
+        f'stackl-client==0.4.0', 'pyYAML==5.4.1', 'Click==7.0',
         'mergedeep==1.3.0', 'tabulate==0.8.6', 'glom==19.10.0'
     ],
     entry_points='''

stackl/core/core/agent_broker/agent_task_broker.py

@@ -14,7 +14,7 @@
 
 
 async def create_service(action, redis, stack_instance, to_be_deleted,
-                         force_delete, service_name, service):
+                         force_delete, service_name, service, index):
     opa_broker_factory = OPABrokerFactory()
     opa_broker = opa_broker_factory.get_opa_broker()
     success = True
@@ -43,6 +43,10 @@ async def create_service(action, redis, stack_instance, to_be_deleted,
     if action == "delete":
         functional_requirements = reversed(functional_requirements)
     for fr in functional_requirements:
+        stack_instance = document_manager.get_stack_instance(stack_instance.name)
+        service = stack_instance.services[service_name][index]
+        logger.debug(
+            f"Debug for service'{service}'")
         fr_doc = document_manager.get_functional_requirement(fr)
         fr_jobs = []
         infrastructure_target = service.infrastructure_target
@@ -79,6 +83,13 @@ async def create_service(action, redis, stack_instance, to_be_deleted,
                     cloud_provider].playbook_path
             if fr_doc.invocation[cloud_provider].serial:
                 invoc['serial'] = fr_doc.invocation[cloud_provider].serial
+            if fr_doc.invocation[cloud_provider].ansible_role:
+                invoc['ansible_role'] = fr_doc.invocation[cloud_provider].ansible_role
+            if fr_doc.invocation[cloud_provider].connection:
+                invoc['connection'] = fr_doc.invocation[cloud_provider].connection
+            if fr_doc.invocation[cloud_provider].wait_for_port:
+                invoc['wait_for_port'] = fr_doc.invocation[cloud_provider].wait_for_port
+            invoc['gather_facts'] = fr_doc.invocation[cloud_provider].gather_facts
         invoc['service'] = service_name
         invoc["hosts"] = service.hosts
         logger.debug("Appending job")
@@ -135,15 +146,14 @@ async def create_job_per_service(services,
                                  force_delete=False):
     success = True
     stage_jobs = []
-    document_mananger = get_document_manager()
     # do stages
     if stack_instance.stages:
         for stage in stack_instance.stages:
             for service_name in stage.services:
-                for service in stack_instance.services[service_name]:
+                for idx, service in enumerate(stack_instance.services[service_name]):
                     success = create_service(action, redis, stack_instance,
                                              to_be_deleted, force_delete,
-                                             service_name, service)
+                                             service_name, service, idx)
                     stage_jobs.append(asyncio.create_task(success))
 
             await asyncio.gather(*stage_jobs)
@@ -152,10 +162,10 @@ async def create_job_per_service(services,
     # not using stages
     else:
         for service_name, service_list in services.items():
-            for service in service_list:
+            for idx, service in enumerate(service_list):
                 success = await create_service(action, redis, stack_instance,
                                                to_be_deleted, force_delete,
-                                               service_name, service)
+                                               service_name, service, idx)
 
     return success
 

stackl/core/core/handler/stack_handler.py

@@ -371,14 +371,17 @@ def add_outputs(self, outputs_update):
                     **service_definition.outputs,
                     **outputs_update.outputs
                 }
+                if "stackl_hosts" in service_definition.outputs:
+                        service_definition.hosts = service_definition.outputs[
+                            "stackl_hosts"]
+                break
+        for _, service_list in stack_instance.services.items():
+            for service_definition in service_list:
                 service_definition.provisioning_parameters = {
                     **service_definition.provisioning_parameters,
                     **stack_instance.instance_outputs,
                     **service_definition.outputs
                 }
-                if "stackl_hosts" in service_definition.outputs:
-                            service_definition.hosts = service_definition.outputs[
-                                "stackl_hosts"]
 
         return stack_instance
 
@@ -644,14 +647,12 @@ def _update_infr_capabilities(
             else:
                 infr_target_cloud_provider = "generic"
 
-            infr_target_packages = environment.packages + location.packages + zone.packages
             infr_target_key = ".".join(
                 [environment.name, location.name, zone.name])
             infr_targets_capabilities[
                 infr_target_key] = StackInfrastructureTarget(
                     provisioning_parameters=infr_target_capability,
                     tags=infr_target_tags,
-                    packages=infr_target_packages,
                     resources=infr_target_resources,
                     secrets=infr_target_secrets,
                     policies=infr_target_policies,

stackl/core/core/models/configs/functional_requirement_model.py

@@ -1,7 +1,7 @@
 """
 Module containing functional requirement classes
 """
-from typing import Dict, Optional
+from typing import Any, Dict, Optional, Union
 
 from pydantic import BaseModel  # pylint: disable=E0611 #error in pylint
 
@@ -15,7 +15,11 @@ class Invocation(BaseModel):
     image: str
     before_command: Optional[str] = ""
     playbook_path: str = ""
+    ansible_role: str = ""
+    gather_facts: bool = True
     serial: int = 10
+    connection: str = "ssh"
+    wait_for_port: Any = None
 
 
 class FunctionalRequirement(BaseDocument):

stackl/core/core/models/configs/infrastructure_base_document.py

@@ -18,6 +18,5 @@ class InfrastructureBaseDocument(BaseModel):
     outputs: dict = {}
     resources: dict = {}
     policies: dict = {}
-    packages: List[str] = []
     tags: dict = {}
     agent: str = ""

stackl/core/core/models/items/stack_infrastructure_target_model.py

@@ -16,5 +16,4 @@ class StackInfrastructureTarget(BaseModel):
     resources: Dict[str, str] = None
     policies: Dict[str, Any] = None
     agent: str = ""
-    packages: List[str] = None
     tags: Dict[str, str] = None

stackl/core/core/models/items/stack_instance_service_model.py

@@ -29,7 +29,6 @@ class StackInstanceService(BaseModel):
     agent: str = ""
     opa_outputs: Dict[str, Any] = {}
     outputs: Dict[str, Any] = {}
-    packages: List[str] = []
     tags: Dict[str, str] = {}
     service: str = ""
 

stackl/core/core/opa_broker/opa_broker.py

@@ -25,8 +25,6 @@ def convert_sit_to_opa_data(sit_doc: StackInfrastructureTemplate):
         target_data = {
             "resources":
             sit_doc.infrastructure_capabilities[target].resources,
-            "packages":
-            sit_doc.infrastructure_capabilities[target].packages,
             "tags":
             sit_doc.infrastructure_capabilities[target].tags,
             "params":

stackl/core/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "core"
-version = "0.3.4dev"
+version = "0.4.0"
 description = ""
 authors = ["Frederic <[email protected]>"]
 

stackl/core/tests/test_infrastructure_base.py

@@ -13,9 +13,6 @@
     },
     "resources": {},
     "policies": {},
-    "packages": [
-        "test",
-    ],
     "outputs": {},
     "cloud_provider": "aws",
     "tags": {
@@ -30,7 +27,6 @@
 
 environment_update = environment.copy()
 environment_update['description'] = "Update description"
-environment_update['packages'] = ['update']
 environment_update['secrets'] = {}
 environment_update['tags'] = {'environment': 'prod'}
 environment_update['params'] = {'update': 'update'}
@@ -92,7 +88,6 @@ def test_get_environments_expect_none():
     'outputs': {},
     'resources': {},
     'policies': {},
-    'packages': [],
     'tags': {},
     'agent': '',
     'description': 'Belgium DC'
@@ -158,7 +153,6 @@ def test_get_locations_expect_none():
     'outputs': {},
     'resources': {},
     'policies': {},
-    'packages': [],
     'tags': {},
     'agent': '',
     'description': 'Dome AWS zone'