From 1a4d95eaa1786048d818f3a883bc5798d488d7cf Mon Sep 17 00:00:00 2001 From: "michael.osl@kununu.com" Date: Thu, 11 May 2017 14:27:31 +0200 Subject: [PATCH 1/4] Work on first version of task runner --- .gitignore | 2 ++ ecstaskrunner/__init__.py | 44 +++++++++++++++++++++++++++ ecstaskrunner/container.py | 61 ++++++++++++++++++++++++++++++++++++++ ecstaskrunner/task.py | 46 ++++++++++++++++++++++++++++ setup.py | 13 ++++++++ 5 files changed, 166 insertions(+) create mode 100644 ecstaskrunner/__init__.py create mode 100644 ecstaskrunner/container.py create mode 100644 ecstaskrunner/task.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 72364f9..607518f 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ ENV/ # Rope project settings .ropeproject + +*.swp diff --git a/ecstaskrunner/__init__.py b/ecstaskrunner/__init__.py new file mode 100644 index 0000000..596aae5 --- /dev/null +++ b/ecstaskrunner/__init__.py @@ -0,0 +1,44 @@ +import boto3 +import sys +import time +import logging +import botocore.errorfactory +import ecstaskrunner + +def run_task(**kwargs): + client = boto3.client('ecs') + + response = client.run_task(**kwargs) + + logger = logging.getLogger('ecstaskrunner') + + taskResponse = response['tasks'][0] + + taskId = taskResponse['taskArn'][taskResponse['taskArn'].rfind("/")+1:] + logger.debug(taskId) + + task = ecstaskrunner.task.Task(taskResponse['clusterArn'], taskId) + + logger.debug("task is pending") + while task.isPending(): + time.sleep(1) + + while task.isRunning(): + for container in task.containers: + for log_event in task.containers[container].get_log_events(): + logger.info("%s: %s" % (container, log_event)) + logger.debug("task status: %s" % task.getLastStatus()) + + exitCode = 0 + + for container in task.describeTask()['containers']: + if 'reason' in container: + logger.warn("%s failed: %s" % (container['name'], container['reason'])) + exitCode = 1 + continue + + logger.info("%s exited with code %d" % (container['name'], container['exitCode'])) + if container['exitCode'] != 0: + exitCode = 2 + + return exitCode diff --git a/ecstaskrunner/container.py b/ecstaskrunner/container.py new file mode 100644 index 0000000..6f0e790 --- /dev/null +++ b/ecstaskrunner/container.py @@ -0,0 +1,61 @@ +import boto3 +import logging +import datetime + +class Container: + def __init__(self, taskId, config, definition): + self.taskId = taskId + self.config = config + self.definition = definition + self.name = config['name'] + self.startTime = None + self.logger = logging.getLogger("Container") + + def get_log_events(self): + logConfig = self.definition['logConfiguration'] + tasklogger = logging.getLogger(self.name) + + logs = boto3.client('logs') + + logStreamName = '%s/%s/%s' % ( + logConfig['options']['awslogs-stream-prefix'], + self.name, + self.taskId + ) + nextToken = False + while True: + a = { + 'logGroupName': logConfig['options']['awslogs-group'], + 'logStreamName': logStreamName, + 'startFromHead': True, + 'limit': 2 + } + + if nextToken: + a['nextToken'] = nextToken + else: + if self.startTime: + a['startTime'] = self.startTime + + try: + response = logs.get_log_events(**a) + except Exception as e: + # todo not sure why i cannot check for the class directly + if e.__class__.__name__ == 'ResourceNotFoundException': + self.logger.warn(e) + return + raise e + for event in response['events']: + yield "[%s] %s" % ( + datetime.datetime.fromtimestamp( + event['timestamp']/1000 + ), + event['message']) + self.startTime = event['timestamp']+1 + + if len(response['events']) != a['limit']: + self.logger.debug("[EOS]") + break + + nextToken = response['nextForwardToken'] + diff --git a/ecstaskrunner/task.py b/ecstaskrunner/task.py new file mode 100644 index 0000000..572e992 --- /dev/null +++ b/ecstaskrunner/task.py @@ -0,0 +1,46 @@ +import boto3 +import logging +from ecstaskrunner.container import Container + +class Task: + def __init__(self, cluster, taskId): + self.cluster = cluster + self.taskId = taskId + self.containers = {} + self.client = boto3.client('ecs') + self.logger = logging.getLogger('Task') + + task = self.describeTask() + + if not task: + self.logger.warn("Task with id %s does not exist" % taskId) + return + + self.taskDefinitionArn = task['taskDefinitionArn'] + containers = task['containers'] + response = self.client.describe_task_definition(taskDefinition=self.taskDefinitionArn) + containerDefinitions = response['taskDefinition']['containerDefinitions'] + + self.logger.debug("task definition arn: %s" % self.taskDefinitionArn) + + for container in self.describeTask()['containers']: + self.containers[container['name']] = Container( + self.taskId, + container, + [x for x in containerDefinitions if x['name'] == container['name']][0] + ) + + def describeTask(self): + response = self.client.describe_tasks(cluster=self.cluster, tasks=[self.taskId]) + if len(response['tasks']) != 1: + return None + return response['tasks'][0] + + def getLastStatus(self): + return self.describeTask()['lastStatus'] + + def isRunning(self): + return self.getLastStatus() == "RUNNING" + + def isPending(self): + return self.getLastStatus() == "PENDING" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..31b8a76 --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup(name='ecstaskrunner', + version='0.0.1', + description='Helper tools to run a task on AWS ECS and follow the logs', + author='Michael Osl', + author_email='moee@users.noreply.github.com', + url='https://github.com/moee/ecstaskrunner', + packages=['ecstaskrunner'], +) + From 55e11156243e9669bb3886b590c8395c718b392f Mon Sep 17 00:00:00 2001 From: "michael.osl@kununu.com" Date: Thu, 11 May 2017 14:37:02 +0200 Subject: [PATCH 2/4] Require boto3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 31b8a76..f80bcc0 100644 --- a/setup.py +++ b/setup.py @@ -9,5 +9,6 @@ author_email='moee@users.noreply.github.com', url='https://github.com/moee/ecstaskrunner', packages=['ecstaskrunner'], + install_requires=['boto3'] ) From a462b4862038d0fbf117c22c24b9259140d4b4cd Mon Sep 17 00:00:00 2001 From: "michael.osl@kununu.com" Date: Thu, 11 May 2017 14:41:20 +0200 Subject: [PATCH 3/4] Fix import --- ecstaskrunner/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecstaskrunner/__init__.py b/ecstaskrunner/__init__.py index 596aae5..93175bf 100644 --- a/ecstaskrunner/__init__.py +++ b/ecstaskrunner/__init__.py @@ -4,6 +4,7 @@ import logging import botocore.errorfactory import ecstaskrunner +import ecstaskrunner.task def run_task(**kwargs): client = boto3.client('ecs') From 51345526a2b2b6f5135f7da99855dc0b419083b4 Mon Sep 17 00:00:00 2001 From: Michael Osl Date: Thu, 11 May 2017 16:26:24 +0200 Subject: [PATCH 4/4] Write Readme with examples --- README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/README.md b/README.md index 4617249..1ac1f6a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ # ecs_task_runner Helper tools to run a task on AWS ECS and follow the logs + +# Why? +The use case why I created this script is that we want to run ECS tasks in Jenkins and directly see the output of the tasks that are being run. This is very cumbersome to achieve using the `aws cli`. Hence this library. + +# Installation +Until this package is distributed as pip package, you have to install it directly from this repository: + +``` +pip install git+https://github.com/moee/ecs_task_runner@0.0.1 +``` + +# Usage +## Example 1: Jenkins Integration + +```sh +#!/bin/sh +pip install git+https://github.com/moee/ecs_task_runner@0.0.1 + +python << END +import ecstaskrunner, sys, logging + +logging.basicConfig() +logging.getLogger('ecstaskrunner').setLevel(logging.INFO) + +sys.exit( + ecstaskrunner.run_task( + cluster="YOUR-CLUSTER-NAME", + taskDefinition='YOUR-TASK-DEFINITION', + ) +) +END +``` +This runs the task named `YOUR-TASK-DEFINITION` on the cluster `YOUR-CLUSTER-NAME`, displays all the output (Note: this only works if the container definition uses the awslogs driver) and waits for the task to stop. Only if all containers have stopped and exited with `0` the job will be marked as success. + +## Example 2: Get the log output of a task + +```python +import ecstaskrunner +task = ecstaskrunner.task.Task(cluster='YOUR-CLUSTER-NAME', taskId='YOUR-TASK-ID') +for container in task.containers: + for line in task.containers[container].get_log_events(): + print "%s: %s" % (container, line) +```