-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
70 changed files
with
1,290 additions
and
300 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| name: 'Tag or Release' | ||
| description: 'tag or release from a given branch' | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - run: ${{ github.action_path }}/tag_or_release.sh | ||
| shell: bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | ||
| git config --local user.name "github-actions[bot]" | ||
|
|
||
| if [[ "$tag_or_release" == "tag" ]]; then | ||
| git checkout ${source_branch} | ||
| git tag -f "${name}" -m "tagged ${source_branch} to ${name} via manual github action" | ||
| # will fail if tag already exists; intentional | ||
| git push origin ${name} | ||
|
|
||
| elif [[ "$tag_or_release" == "release" ]]; then | ||
| echo ${token} | gh auth login --with-token | ||
| gh release create ${name} -F changelog.md --target ${source_branch} --title ${name} | ||
|
|
||
| else | ||
| echo "bad input" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| name: tag-or-release | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| tag_or_release: | ||
| description: 'must be string of either "tag" or "release"' | ||
| required: true | ||
| default: 'tag' | ||
| name: | ||
| description: 'the tag or release name, i.e. v1.0.0' | ||
| required: true | ||
| source_branch: | ||
| description: 'the branch to tag or release' | ||
| required: true | ||
| default: "main" | ||
|
|
||
| jobs: | ||
| tag_or_release: | ||
| runs-on: ubuntu-latest | ||
| name: tag or release the given branch with the given name | ||
| steps: | ||
| - name: checkout | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: release | ||
| id: release | ||
| uses: ./.github/actions/release | ||
| env: | ||
| tag_or_release: "${{ github.event.inputs.tag_or_release }}" | ||
| name: "${{ github.event.inputs.name }}" | ||
| source_branch: "${{ github.event.inputs.source_branch }}" | ||
| token: "${{ secrets.GITHUB_TOKEN }}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| Content-Type: multipart/mixed; boundary="==BOUNDARY==" | ||
| MIME-Version: 1.0 | ||
|
|
||
| --==BOUNDARY== | ||
| MIME-Version: 1.0 | ||
| Content-Type: text/x-shellscript; charset="us-ascii" | ||
|
|
||
| #!/bin/bash -ex | ||
| exec &> >(while read line; do echo "$(date +'%Y-%m-%dT%H.%M.%S%z') $line" >> /var/log/user-data.log; done;) | ||
| # ensures instance will shutdown even if we don't reach the end | ||
| shutdown -h +20 | ||
| log_stream="`date +'%Y-%m-%dT%H.%M.%S%z'`" | ||
| sleep 5 | ||
|
|
||
| cat << EOF > /home/ec2-user/log_listener.py | ||
| import boto3 | ||
| import time | ||
| import sys | ||
| from datetime import datetime | ||
| client = boto3.client('logs') | ||
| log_group = sys.argv[1] | ||
| log_stream = sys.argv[2] | ||
| pushed_lines = [] | ||
| while True: | ||
| response = client.describe_log_streams( | ||
| logGroupName=log_group, | ||
| logStreamNamePrefix=log_stream | ||
| ) | ||
| try: | ||
| nextToken = response['logStreams'][0]['uploadSequenceToken'] | ||
| except KeyError: | ||
| nextToken = None | ||
| with open("/var/log/user-data.log", 'r') as f: | ||
| lines = f.readlines() | ||
| new_lines = [] | ||
| for line in lines: | ||
| if line in pushed_lines: | ||
| continue | ||
| timestamp = line.split(" ")[0].strip() | ||
| try: | ||
| dt = datetime.strptime(timestamp, "%Y-%m-%dT%H.%M.%S%z") | ||
| dt_ts = int(dt.timestamp())*1000 #milliseconds | ||
| if nextToken is None: | ||
| response = client.put_log_events( | ||
| logGroupName = log_group, | ||
| logStreamName = log_stream, | ||
| logEvents = [ | ||
| { | ||
| 'timestamp': dt_ts, | ||
| 'message': line | ||
| } | ||
| ] | ||
| ) | ||
| nextToken = response['nextSequenceToken'] | ||
| else: | ||
| response = client.put_log_events( | ||
| logGroupName = log_group, | ||
| logStreamName = log_stream, | ||
| logEvents = [ | ||
| { | ||
| 'timestamp': dt_ts, | ||
| 'message': line | ||
| } | ||
| ], | ||
| sequenceToken=nextToken | ||
| ) | ||
| nextToken = response['nextSequenceToken'] | ||
| except Exception as e: | ||
| # print(e) | ||
| continue | ||
| pushed_lines.append(line) | ||
| time.sleep(0.21) #AWS throttles at 5 calls/second | ||
| time.sleep(2) | ||
| EOF | ||
|
|
||
| echo BEGIN | ||
| pwd | ||
| date '+%Y-%m-%d %H:%M:%S' | ||
|
|
||
| yum install -y -q gcc libpng-devel libjpeg-devel unzip yum-utils | ||
| yum update -y -q && yum upgrade -q | ||
| cd /home/ec2-user | ||
| curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | ||
| unzip -qq awscliv2.zip | ||
| ./aws/install --update | ||
| curl -s "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb" | ||
| mkdir /home/ec2-user/.aws | ||
| yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo | ||
| yum install terraform-0.15.4-1 -y -q | ||
| yum install git -y -q | ||
| chown -R ec2-user:ec2-user /home/ec2-user/ | ||
|
|
||
| echo "export REQUESTS_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" >> /home/ec2-user/.bashrc | ||
| echo "export CURL_CA_BUNDLE=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" >> /home/ec2-user/.bashrc | ||
| mkdir -p /usr/lib/ssl | ||
| mkdir -p /etc/ssl/certs | ||
| mkdir -p /etc/pki/ca-trust/extracted/pem | ||
| ln -s /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/certs/ca-certificates.crt | ||
| ln -s /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /usr/lib/ssl/cert.pem | ||
|
|
||
| yum install python3 -y -q | ||
|
|
||
| sudo -i -u ec2-user bash << EOF | ||
| mkdir ~/bin ~/tmp | ||
| cd ~/tmp | ||
| curl -s -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash | ||
| bash ~/.nvm/nvm.sh | ||
| source ~/.bashrc | ||
| nvm install node | ||
| npm config set registry http://registry.npmjs.org/ | ||
| npm config set cafile /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem | ||
| npm install -g [email protected] | ||
| python3 -m pip install -q --upgrade pip && python3 -m pip install boto3 -q | ||
| cd ~ | ||
| rm -rf ~/tmp | ||
| EOF | ||
|
|
||
| chown -R ec2-user:ec2-user /home/ec2-user/ | ||
|
|
||
| echo "export ADMIN_ARN=${admin_arn}" >> /home/ec2-user/.bashrc | ||
| echo "export AWS_DEFAULT_REGION=us-east-1" >> /home/ec2-user/.bashrc | ||
| echo "export aws_env=${environment}" >> /home/ec2-user/.bashrc | ||
|
|
||
| # get cloudwatch logging going | ||
| sudo -i -u ec2-user bash << EOF | ||
| cd /home/ec2-user | ||
| source .bashrc | ||
| aws logs create-log-stream --log-group-name "${log_group}" --log-stream-name $log_stream | ||
| python3 /home/ec2-user/log_listener.py "${log_group}" $log_stream & | ||
| EOF | ||
|
|
||
| # calcloud checkout, need right tag | ||
| cd /home/ec2-user | ||
| mkdir ami_rotate && cd ami_rotate | ||
| git clone https://github.com/spacetelescope/calcloud.git | ||
| cd calcloud | ||
| git remote set-url origin DISABLED --push | ||
| git fetch | ||
| git fetch --all --tags && git checkout tags/v${calcloud_ver} && cd .. | ||
| git_exit_status=$? | ||
| if [[ $git_exit_status -ne 0 ]]; then | ||
| # try without the v | ||
| cd calcloud && git fetch --all --tags && git checkout tags/${calcloud_ver} && cd .. | ||
| git_exit_status=$? | ||
| fi | ||
| if [[ $git_exit_status -ne 0 ]]; then | ||
| echo "could not checkout ${calcloud_ver}; exiting" | ||
| exit 1 | ||
| fi | ||
|
|
||
| sudo -i -u ec2-user bash << EOF | ||
| cd /home/ec2-user | ||
| source .bashrc | ||
| cd ami_rotate/calcloud/terraform | ||
| ./deploy_ami_rotate.sh | ||
| EOF | ||
|
|
||
| sleep 120 #let logs catch up | ||
|
|
||
| shutdown -h now | ||
|
|
||
| --==BOUNDARY==-- |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| import sys | ||
| import json | ||
| from collections import OrderedDict | ||
| from datetime import datetime | ||
|
|
||
| response = json.loads(str(sys.argv[1])) | ||
| images = response["Images"] | ||
| image_name_filter = sys.argv[2] | ||
|
|
||
| stsciLinux2Ami = {} | ||
| for image in images: | ||
| creationDate = image["CreationDate"] | ||
| imageId = image["ImageId"] | ||
| name = image["Name"] | ||
| # Only look at particular AMIs | ||
| if name.startswith(image_name_filter): | ||
| stsciLinux2Ami.update({creationDate: imageId}) | ||
| # Order the list most recent date first | ||
| orderedAmi = OrderedDict( | ||
| sorted(stsciLinux2Ami.items(), key=lambda x: datetime.strptime(x[0], "%Y-%m-%dT%H:%M:%S.%f%z"), reverse=True) | ||
| ) | ||
| # Print first element in the ordered dict | ||
| print(list(orderedAmi.values())[0]) |
Oops, something went wrong.