Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Issue-365: Use amazon linux 2 AMI for ECS (if none specified) #377

Open
wants to merge 11 commits into
base: develop
Choose a base branch
from
Open

Issue-365: Use amazon linux 2 AMI for ECS (if none specified) #377

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c0b9701
use amazon linux 2 AMI for ECS (if none specified)
AndreyMarchuk Oct 23, 2018
1c6851c
Merge branch 'develop' into feature/issue-365
cplee Oct 24, 2018
36f8880
switch to new cloudwatch agent for logs
AndreyMarchuk Oct 25, 2018
d8e97f2
typo fix
AndreyMarchuk Oct 25, 2018
bb2245d
Merge branch 'develop' into feature/issue-365
cplee Oct 26, 2018
45a1e52
Merge branch 'develop' into feature/issue-365
cplee Oct 31, 2018
49a6f8a
Merge branch 'develop' into feature/issue-365
cplee Nov 21, 2018
0a45c2f
Merge branch 'develop' into feature/issue-365
cplee Dec 5, 2018
062ed80
clean commented code
AndreyMarchuk Feb 12, 2019
7ad64b2
clean commented code
AndreyMarchuk Feb 12, 2019
aababa2
Merge branch 'develop' into feature/issue-365
cplee Feb 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 146 additions & 62 deletions templates/assets/cloudformation/env-ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,29 +262,33 @@ Resources:
amazonLinux:
packages:
yum:
awslogs: []
jq: []
aws-cli: []
files:
"/etc/awslogs/awscli.conf":
content: !Sub |
[plugins]
cwlogs = cwlogs
[default]
region = ${AWS::Region}
awscli: []
# files:
# "/etc/awslogs/awscli.conf":
# content: !Sub |
# [plugins]
# cwlogs = cwlogs
# [default]
# region = ${AWS::Region}
commands:
01_configure_ssm_agent:
command: !Sub |
#!/bin/bash
sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
02_configure_cloudwatch_agent:
command: !Sub |
#!/bin/bash
sudo yum install -y https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new CW agent is known to have issues with auto rotation of logs. Have you tested that ? The advantage of using the new unified agent is that it provides metrics and logging in one POST request, but are those metrics being used? It will just incur more cost in case it is left enabled without actually consuming them for alarms or monitoring.

sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s
services:
sysvinit:
awslogs:
enabled: 'true'
ensureRunning: 'true'
files:
- "/etc/awslogs/awslogs.conf"
- "/etc/awslogs/etc/proxy.conf"
# awslogs:
# enabled: 'true'
# ensureRunning: 'true'
# files:
# - "/etc/awslogs/awslogs.conf"
# - "/etc/awslogs/etc/proxy.conf"
cfn-hup:
enabled: 'true'
ensureRunning: 'true'
Expand All @@ -308,61 +312,141 @@ Resources:
path=Resources.ContainerInstances.Metadata.AWS::CloudFormation::Init
action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource ContainerInstances --configsets ${ImageOsType} --region ${AWS::Region}
runas=root
"/etc/awslogs/etc/proxy.conf":
content: !Sub |
HTTP_PROXY=http://${HttpProxy}/
HTTPS_PROXY=http://${HttpProxy}/
"/etc/awslogs/awscli.conf":
content: !Sub |
[plugins]
cwlogs = cwlogs
[default]
region = ${AWS::Region}
"/etc/awslogs/awslogs.conf":
"/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json":
content: !Sub |
[general]
state_file = /var/lib/awslogs/agent-state
{
"agent": {
"metrics_collection_interval": 10,
"region": "${AWS::Region}",
"logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
},
"logs": {
"logs_collected": {
"files": {
"collect_list": [
{
"file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/cloudwatch-agent/{instance_id}",
"timezone": "UTC"
},
{
"file_path": "/var/log/dmesg",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/dmesg/{instance_id}",
"timezone": "UTC"
},
{
"file_path": "/var/log/messages",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/messages/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%b %d %H:%M:%S"
},
{
"file_path": "/var/log/docker",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/docker/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%Y-%m-%dT%H:%M:%S"
},
{
"file_path": "/var/log/ecs/*",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/ecs/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%Y-%m-%dT%H:%M:%SZ"
},
{
"file_path": "/var/log/cloud-init*",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/cloud-init/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%Y-%m-%dT%H:%M:%SZ"
},
{
"file_path": "/var/log/cfn-init*",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/cfn-init/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
},
{
"file_path": "/var/log/amazon/ssm/*",
"log_group_name": "${AWS::StackName}",
"log_stream_name": "instance/ssm/{instance_id}",
"timezone": "UTC",
"timestamp_format": "%Y-%m-%d %H:%M:%S"
}
]
}
},
"log_stream_name": "instance/other/{instance_id}"
}
}
# "/opt/aws/amazon-cloudwatch-agent/etc/common-config.toml":
# content: !Sub |
# # todo: add proxy config
# [proxy]
# http_proxy = "{http_url}"
# https_proxy = "{https_url}"
# no_proxy = "{domain}"

# "/etc/awslogs/etc/proxy.conf":
# content: !Sub |
# HTTP_PROXY=http://${HttpProxy}/
# HTTPS_PROXY=http://${HttpProxy}/
# "/etc/awslogs/awscli.conf":
# content: !Sub |
# [plugins]
# cwlogs = cwlogs
# [default]
# region = ${AWS::Region}
# "/etc/awslogs/awslogs.conf":
# content: !Sub |
# [general]
# state_file = /var/lib/awslogs/agent-state

[dmesg]
file = /var/log/dmesg
log_group_name = ${AWS::StackName}
log_stream_name = instance/dmesg/{instance_id}
# [dmesg]
# file = /var/log/dmesg
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/dmesg/{instance_id}

[messages]
file = /var/log/messages
log_group_name = ${AWS::StackName}
log_stream_name = instance/messages/{instance_id}
datetime_format = %b %d %H:%M:%S
# [messages]
# file = /var/log/messages
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/messages/{instance_id}
# datetime_format = %b %d %H:%M:%S

[docker]
file = /var/log/docker
log_group_name = ${AWS::StackName}
log_stream_name = instance/docker/{instance_id}
datetime_format = %Y-%m-%dT%H:%M:%S.%f
# [docker]
# file = /var/log/docker
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/docker/{instance_id}
# datetime_format = %Y-%m-%dT%H:%M:%S.%f

[ecs]
file = /var/log/ecs/*
log_group_name = ${AWS::StackName}
log_stream_name = instance/ecs/{instance_id}
datetime_format = %Y-%m-%dT%H:%M:%SZ
# [ecs]
# file = /var/log/ecs/*
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/ecs/{instance_id}
# datetime_format = %Y-%m-%dT%H:%M:%SZ

[cloud-init]
file = /var/log/cloud-init*
log_group_name = ${AWS::StackName}
log_stream_name = instance/cloud-init/{instance_id}
datetime_format = %Y-%m-%dT%H:%M:%SZ
# [cloud-init]
# file = /var/log/cloud-init*
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/cloud-init/{instance_id}
# datetime_format = %Y-%m-%dT%H:%M:%SZ

[cfn-init]
file = /var/log/cfn-init*
log_group_name = ${AWS::StackName}
log_stream_name = instance/cfn-init/{instance_id}
datetime_format = %Y-%m-%d %H:%M:%S
# [cfn-init]
# file = /var/log/cfn-init*
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/cfn-init/{instance_id}
# datetime_format = %Y-%m-%d %H:%M:%S

[ssm]
file = /var/log/amazon/ssm/*
log_group_name = ${AWS::StackName}
log_stream_name = instance/ssm/{instance_id}
datetime_format = %Y-%m-%d %H:%M:%S
# [ssm]
# file = /var/log/amazon/ssm/*
# log_group_name = ${AWS::StackName}
# log_stream_name = instance/ssm/{instance_id}
# datetime_format = %Y-%m-%d %H:%M:%S

commands:
01_add_instance_to_cluster:
Expand Down
2 changes: 1 addition & 1 deletion workflows/environment_upsert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

var ecsImageOwner = "amazon"
var ecsImagePattern = "amzn-ami-*-amazon-ecs-optimized"
var ecsImagePattern = "amzn2-ami-ecs-hvm-*-x86_64-ebs"
var eksImageOwner = "602401143452"
var eksImagePattern = "amazon-eks-node-v*"
var ec2ImageOwner = "amazon"
Expand Down