Merge pull request #13 from adobe/read_ssm

Retrieve secrets from AWS SSM

README.md

@@ -138,18 +138,11 @@ $ aws configure --profile aws_account_name
 ## Azure
 TBD
 
-## SKMS
-Create a file in `~/.skms/credentials.yaml` which looks like the following:
-```yaml
-endpoint: "api.skms.mycompany.com"
-username: <username>
-password: <password>
-```
+## Examples
 
-## Example
-
-See `examples/` folder:
+See [examples/](https://github.com/adobe/ops-cli/tree/master/examples) folder:
 - cassandra-stress - n-node cassandra cluster used for stress-testing; a basic stress profile is included
+- spin up a Kubernetes clsuter
 - distinct `ops` features
 
 ## Usage help
@@ -526,14 +519,11 @@ optional arguments:
         ops clusters/centos7.yaml packer build
 ```
 
-## Development
-
-### Running tests
+## Secrets Management
 
-- docker: `buildrunner -f buildrunner.yaml`
-- on your machine: `py.test tests`
+There are cases where you need to reference sensitive data in your `cluster.yaml` file (credentials, passwords, tokens etc). Given that the cluster configuration file can be stored in a version control system (such as Git), the best practice is to not put sensitive data in the file itself. Instead, we can use `ops-cli` to fetch the desired credentials from a secrets manager such as Vault or Amazon SSM, at runtime.
 
-## Secrets Management
+### Vault
 
 Ops can manage the automatic generation of secrets and their push in Vault, without actually persisting the secrets in the cluster file.
 A cluster file will only need to use a construct like the following:
@@ -548,6 +538,20 @@ Which will translate behind the scenes in :
 This allows us to just refer in cluster files a secret that actually exists in vault and make sure we only generate it once - if it was already created by os or any other system, we will just use what is already there.
 The reference is by means of fixed form jinja call  added to the cluster file, which ends up interpretted later during the templating phase.
 
+### Amazon Secrets Manager (SSM)
+
+Amazon offers the possibility to use their [Secrets Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/what-is-systems-manager.html) in order to manage configuration data such as credentials, passwords and license keys.
+
+We can use `ops-cli` to fetch the sensitive data from SSM, at runtime. Just define this in your cluster configuration file (eg. `mycluster.yaml`).
+
+```
+db_password: "{{ '/my/ssm/path' | read_ssm(aws_profile='myprofile') }}"
+```
+
+`ops-cli` will read the SSM value by running a command similar to: `AWS_PROFILE=aam-npe aws ssm get-parameter --name "/my/ssm/path"  --region us-east-1 --with-decryption`.
+Note that you can specify the AWS region via `read_ssm(aws_profile='myprofile', region_name='us-west-2')`.
+
+
 ## Using jinja2 filters in playbooks and terraform templates
 
 You can register your own jinja2 filters that you can  use in the cluster config file, terraform templates and ansible playbooks
@@ -575,6 +579,20 @@ class FilterModule(object):
 # test_custom_filters: "{{ 'value' | my_filter }}"
 ```
 
+## SKMS
+Create a file in `~/.skms/credentials.yaml` which looks like the following:
+```yaml
+endpoint: "api.skms.mycompany.com"
+username: <username>
+password: <password>
+```
+
+## Development
+
+### Running tests
+
+- on your machine: `py.test tests`
+
 ## Troubleshooting
 
 - Permission issues when installing: you should install the tool in a python virtualenv

requirements.txt

@@ -1,7 +1,7 @@
 simpledi>=0.2
 awscli==1.16.97
 ansible==2.3.1.0
-boto3==1.9.87
+boto3==1.9.91
 boto==2.49.0
 azure-common==1.1.4
 azure==2.0.0rc5

src/ops/ansible/filter_plugins/commonfilters.py

@@ -65,12 +65,12 @@ def flatten_tree(d, parent_key='', sep='/'):
     return dict(items)
 
 def read_vault(secret_path, key='value', fetch_all=False, vault_user=None,vault_url=None, token=None, auto_prompt=True):
-    from ops.simplesecrets import SimpleVault
+    from ops.simplevault import SimpleVault
     sv = SimpleVault(vault_user=vault_user, vault_addr=vault_url, vault_token=token,auto_prompt=auto_prompt)
     return sv.get(path=secret_path, key=key, fetch_all=fetch_all)
 
 def write_vault(secret_path, key='value', data="", vault_user=None, vault_url=None, token=None, auto_prompt=True):
-    from ops.simplesecrets import  SimpleVault
+    from ops.simplevault import  SimpleVault
     sv = SimpleVault(vault_user=vault_user, vault_addr=vault_url, vault_token=token, auto_prompt=auto_prompt)
     new_data = {}
     if isinstance(data, dict):
@@ -82,13 +82,18 @@ def write_vault(secret_path, key='value', data="", vault_user=None, vault_url=No
         return False
     return sv.put(path=secret_path, value=new_data )
 
+def read_ssm(key, aws_profile, region_name='us-east-1'):
+    from ops.simplessm import SimpleSSM
+    ssm = SimpleSSM(aws_profile, region_name)
+    return ssm.get(key)
+
 def managed_vault_secret(secret_path,key='value',
                          policy={},
                          vault_user=None,
                          vault_addr=None,
                          vault_token=None,
                          auto_prompt=True):
-    from ops.simplesecrets import ManagedVaultSecret
+    from ops.simplevault import ManagedVaultSecret
     ms = ManagedVaultSecret(path=secret_path,
                             key=key,
                             policy=policy,
@@ -111,5 +116,6 @@ def filters(self):
             'read_yaml': read_yaml,
             'read_vault': read_vault,
             'write_vault': write_vault,
-            'managed_vault_secret': managed_vault_secret
+            'managed_vault_secret': managed_vault_secret,
+            'read_ssm': read_ssm
         }

src/ops/simplessm.py

@@ -0,0 +1,41 @@
+#Copyright 2019 Adobe. All rights reserved.
+#This file is licensed to you under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License. You may obtain a copy
+#of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software distributed under
+#the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+#OF ANY KIND, either express or implied. See the License for the specific language
+#governing permissions and limitations under the License.
+
+#!/usr/bin/env python
+
+from ops.cli import display
+from botocore.exceptions import ClientError
+import boto3
+import os
+
+class SimpleSSM(object):
+    def __init__(self, aws_profile, region_name):
+        self.initial_aws_profile = os.getenv('AWS_PROFILE', None)
+        self.aws_profile = aws_profile
+        self.region_name = region_name
+
+    def get(self, key):
+        client = self.get_ssm_client()
+        try:
+            return client.get_parameter(Name=key, WithDecryption=True).get("Parameter").get("Value")
+        except ClientError as e:
+            raise Exception('Error while trying to read SSM value for key: %s - %s' % (key, e.response['Error']['Code']))
+        finally:
+            self.release_ssm_client()
+
+    def get_ssm_client(self):
+        os.environ['AWS_PROFILE'] = self.aws_profile
+        return boto3.client('ssm', region_name=self.region_name)
+
+    def release_ssm_client(self):
+        if self.initial_aws_profile is None:
+            del os.environ['AWS_PROFILE']
+        else:
+            os.environ['AWS_PROFILE'] = self.initial_aws_profile