diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 0000000..a126861 --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,14 @@ +addReviewers: true +addAssignees: true +reviewers: + - okgolove + - rfvermut +numberOfReviewers: 0 +assignees: + - okgolove + - rfvermut +numberOfAssignees: 1 + + +skipKeywords: + - wip diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml new file mode 100644 index 0000000..2986f2b --- /dev/null +++ b/.github/workflows/assign.yml @@ -0,0 +1,10 @@ +name: 'Auto Assign' +on: pull_request + +jobs: + add-reviews: + runs-on: ubuntu-latest + steps: + - uses: kentaro-m/auto-assign-action@v1.0.1 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7693ebb --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint +on: [push, pull_request] + +jobs: + tflint: + name: TFLint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: TFLint + uses: docker://wata727/tflint + + fmt: + name: Code Format + runs-on: ubuntu-latest + container: + image: hashicorp/terraform:latest + steps: + - uses: actions/checkout@master + - run: terraform fmt --recursive -check=true + + validate: + name: Validate + runs-on: ubuntu-latest + container: + image: hashicorp/terraform:latest + steps: + - uses: actions/checkout@master + - name: Validate Code + env: + AWS_REGION: 'us-east-1' + TF_WARN_OUTPUT_ERRORS: 1 + run: | + terraform init + terraform validate diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f21c517 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +- repo: git://github.com/antonbabenko/pre-commit-terraform + rev: v1.19.0 + hooks: + - id: terraform_fmt + - id: terraform_docs + - id: terraform_validate + - id: terraform_tflint diff --git a/README.md b/README.md new file mode 100644 index 0000000..011fada --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:-----:| +| bastion\_instance\_size | n/a | `string` | `"t3.nano"` | no | +| bastion\_ssh\_keys | n/a | `list(string)` | n/a | yes | +| config\_output\_path | n/a | `any` | n/a | yes | +| eks\_authorized\_roles | n/a | `list(string)` | `[]` | no | +| extra\_policy\_arn | n/a | `string` | `"arn:aws:iam::aws:policy/AmazonS3FullAccess"` | no | +| instance\_types | n/a | `list(string)` | n/a | yes | +| ip\_whitelist | n/a | `list(string)` | `[]` | no | +| key\_name | n/a | `string` | n/a | yes | +| kubectl\_assume\_role | n/a | `string` | `""` | no | +| project\_fqdn | n/a | `string` | n/a | yes | +| project\_prefix | n/a | `string` | n/a | yes | +| project\_rev\_fqdn | n/a | `string` | n/a | yes | +| spot\_price | n/a | `string` | `""` | no | +| vpc\_cidr | n/a | `string` | `"172.31.0.0/16"` | no | +| worker\_groups | n/a | `list` |
[
{
"instance_type": "t3.large"
},
{
"instance_type": "t3.2xlarge"
}
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| bastion | How to reach bastion | +| cluster\_name | n/a | +| eks\_cluster | n/a | +| kubeconfig\_filename | n/a | +| vpc | n/a | +| whitelist\_sg\_id | n/a | + diff --git a/bastions.tf b/bastions.tf new file mode 100644 index 0000000..c2d7e0c --- /dev/null +++ b/bastions.tf @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +data "aws_ami" "amazon-linux" { + most_recent = true + + filter { + name = "name" + values = ["amzn-ami-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + # Amazon + owners = ["137112412989"] +} + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + # Canonical + owners = ["099720109477"] +} + +resource "aws_instance" "bastion" { + ami = data.aws_ami.ubuntu.id + instance_type = var.bastion_instance_size + subnet_id = module.vpc.public_subnets[0] + + vpc_security_group_ids = [ + aws_security_group.bastion_sg.id, + aws_security_group.bastion_incoming_ssh.id, + ] + + key_name = var.key_name + user_data = templatefile("${path.module}/templates/bastion_ssh_keys.sh.tpl", { bastion_ssh_keys = var.bastion_ssh_keys }) + tags = local.bastion_tags + + lifecycle { + ignore_changes = [ + ami, + ] + } +} + +resource "aws_security_group" "allow_ssh_from_bastion" { + name = "${var.project_prefix}-eks-bastion_ssh_access" + description = "Allow SSH from bastion" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + security_groups = [ + aws_security_group.bastion_sg.id, + ] + } + + egress { + from_port = 22 + to_port = 22 + protocol = "tcp" + security_groups = [ + aws_security_group.bastion_sg.id, + ] + } +} + +resource "aws_security_group" "bastion_incoming_ssh" { + name = "${var.project_prefix}-eks-bastion_incoming_ssh" + description = "Allow SSH to bastion from world" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + + cidr_blocks = var.ip_whitelist + } +} + +resource "aws_security_group" "bastion_sg" { + name = "${var.project_prefix}-eks-bastion_sg" + description = "Bastion SG" + vpc_id = module.vpc.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = -1 + + # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to + # force an interpolation expression to be interpreted as a list by wrapping it + # in an extra set of list brackets. That form was supported for compatibility in + # v0.11, but is no longer supported in Terraform v0.12. + # + # If the expression in the following list itself returns a list, remove the + # brackets to avoid interpretation as a list of lists. If the expression + # returns a single list item then leave it as-is and remove this TODO comment. + security_groups = [ + module.eks.worker_security_group_id, + ] + } +} + +output "bastion" { + description = "How to reach bastion" + value = "ubuntu@${aws_instance.bastion.public_ip}" +} diff --git a/eks-module.tf b/eks-module.tf new file mode 100644 index 0000000..c5ad27d --- /dev/null +++ b/eks-module.tf @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +locals { + kubectl_assume_role_args = split(",", var.kubectl_assume_role != "" ? join(",", ["\"-r\"", "\"${var.kubectl_assume_role}\""]) : "", ) + cluster_name = "${var.project_prefix}-eks-cluster" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "6.0.2" + cluster_name = local.cluster_name + cluster_version = "1.14" + tags = local.eks_tags + + cluster_create_timeout = "1h" + cluster_delete_timeout = "1h" + + vpc_id = module.vpc.vpc_id + subnets = [ + module.vpc.private_subnets[0], + module.vpc.private_subnets[1], + ] + cluster_endpoint_public_access = "true" + cluster_endpoint_private_access = "true" + worker_additional_security_group_ids = [ + aws_security_group.whitelist.id, + aws_security_group.allow_ssh_from_bastion.id, + ] + + kubeconfig_aws_authenticator_additional_args = local.kubectl_assume_role_args + + map_roles = [for role in var.eks_authorized_roles : + { + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${role}" + username = "eks-admin:{{SessionName}}" + groups = [ + "system:masters", + ] + }] + + config_output_path = "${var.config_output_path}/" + + worker_groups = flatten([ + for group in var.worker_groups : [ + for subnet in module.vpc.private_subnets : + merge(group, { + subnets = [subnet] + }) + ]]) + + + workers_group_defaults = { + asg_desired_capacity = 1 + asg_max_size = 25 + asg_min_size = 0 + asg_force_delete = true + spot_price = var.spot_price + autoscaling_enabled = true + key_name = var.key_name + enabled_metrics = [ + "GroupInServiceInstances", + "GroupDesiredCapacity", + ] + kubelet_extra_args = "--fail-swap-on=false --eviction-hard=memory.available<500Mi --system-reserved=memory=1Gi" + bootstrap_extra_args = "--enable-docker-bridge true" + pre_userdata = <<-EOF + bash <(curl https://gist.githubusercontent.com/rfvermut/4f141cbdfd107d95018731439ffe737d/raw/001cfdbf532d84c7307be4133883202dbcf96e58/add_swap.sh) 2 + echo "$(jq '."default-ulimits".nofile.Hard=65536 | ."default-ulimits".nofile.Soft=65536 | ."default-ulimits".nofile.Name="NOFILE"' /etc/docker/daemon.json)" > /etc/docker/daemon.json + systemctl restart docker +EOF + + } +} + +// Poor man's money saver +resource "aws_autoscaling_schedule" "tgi_friday" { + count = length(module.eks.workers_asg_names) + + scheduled_action_name = "friday-off" + recurrence = "0 1 * * SAT" + min_size = -1 + max_size = -1 + desired_capacity = 0 + autoscaling_group_name = element(module.eks.workers_asg_names, count.index) +} + +resource "aws_autoscaling_schedule" "monday_im_in_love" { + scheduled_action_name = "monday-on" + recurrence = "0 7 * * MON" + min_size = -1 + max_size = -1 + desired_capacity = 1 + autoscaling_group_name = module.eks.workers_asg_names[0] +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryPowerUser" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + role = module.eks.worker_iam_role_name +} + +// TODO maybe more restrictive +resource "aws_iam_role_policy_attachment" "workers_AmazonRoute53FullAccess" { + policy_arn = "arn:aws:iam::aws:policy/AmazonRoute53FullAccess" + role = module.eks.worker_iam_role_name +} + +resource "aws_iam_role_policy_attachment" "workers_extra_policy" { + policy_arn = var.extra_policy_arn + role = module.eks.worker_iam_role_name +} + +# TODO: use these policies instead of full Route53 access +# +# data "aws_iam_policy_document" "cert-manager-route53" { +# statement { +# actions = [ +# "route53:GetChange" +# ] + +# resources = [ +# "arn:aws:route53:::change/${aws_route53_zone.primary.zone_id}", +# ] +# } + +# statement { +# actions = [ +# "route53:ChangeResourceRecordSets" +# ] + +# resources = [ +# "arn:aws:route53:::hostedzone/${aws_route53_zone.primary.zone_id}", +# ] +# } + +# statement { +# actions = [ +# "route53:ListHostedZonesByName", +# ] + +# resources = [ +# "*", +# ] +# } +# } + +# resource "aws_iam_role_policy" "cert-manager-route53" { +# name = "cert-manager-route53" +# role = "${module.eks.worker_iam_role_name}" +# policy = "${data.aws_iam_policy_document.cert-manager-route53.json}" +# } + +data "aws_iam_policy_document" "eks_default_role" { + statement { + actions = ["sts:AssumeRole"] + + resources = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/EKS-Role-*", + ] + } +} + +resource "aws_iam_role_policy" "eks_default_role" { + name = "eks-default-role" + role = module.eks.worker_iam_role_name + policy = data.aws_iam_policy_document.eks_default_role.json +} diff --git a/k8-sysconfig.tf b/k8-sysconfig.tf new file mode 100644 index 0000000..03ce87d --- /dev/null +++ b/k8-sysconfig.tf @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +resource "kubernetes_service_account" "eks_admin" { + metadata { + name = "eks-admin" + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "eks_admin_cluster_admin" { + depends_on = [ + kubernetes_service_account.eks_admin, + ] + + metadata { + name = "cluster-admin-kube-system-eks-admin" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-admin" + } + + subject { + api_group = "" + kind = "ServiceAccount" + name = "eks-admin" + namespace = "kube-system" + } +} + +resource "null_resource" "gp2" { + provisioner "local-exec" { + command = <<-EOT + kubectl patch --kubeconfig ${var.config_output_path}/kubeconfig_${var.project_prefix}-eks-cluster storageclass gp2 -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +EOT + + } +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..22d50cc --- /dev/null +++ b/main.tf @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +provider "kubernetes" { + version = "~>1.9" + config_path = "${var.config_output_path}/kubeconfig_${local.cluster_name}" +} + +data "aws_caller_identity" "current" { +} + +data "aws_region" "current" { +} + +locals { + aws_region = data.aws_region.current.name + vpc_name = "${var.project_prefix}-vpc" + + vpc_tags = { + Environment = "${var.project_prefix}-infra" + } + + eks_tags = { + Name = "${var.project_prefix}-eks" + Environment = "${var.project_prefix}-infra" + } + + route53_tags = { + Name = "${var.project_prefix}-dns" + Environment = "${var.project_prefix}-infra" + } + + bastion_tags = { + Name = "${var.project_prefix}-eks-bastion" + Environment = "${var.project_prefix}-infra" + } + + // TODO update from API + github_meta_hooks = [ + "192.30.252.0/22", + "185.199.108.0/22", + "140.82.112.0/20", + ] + + // https://confluence.atlassian.com/bitbucket/what-are-the-bitbucket-cloud-ip-addresses-i-should-use-to-configure-my-corporate-firewall-343343385.html + atlassian_inbound = [ + "18.205.93.0/25", + "18.234.32.128/25", + "13.52.5.0/25", + ] +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..d42c756 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +output "kubeconfig_filename" { + value = module.eks.kubeconfig_filename +} + +output "whitelist_sg_id" { + value = aws_security_group.whitelist.id +} + +output "vpc" { + value = module.vpc +} + +output "cluster_name" { + value = local.cluster_name +} + +output "eks_cluster" { + value = module.eks +} diff --git a/templates/bastion_ssh_keys.sh.tpl b/templates/bastion_ssh_keys.sh.tpl new file mode 100644 index 0000000..a3c1a51 --- /dev/null +++ b/templates/bastion_ssh_keys.sh.tpl @@ -0,0 +1,17 @@ +#!/bin/bash -x + +filename=$${0##*/} +echo "`date +%F\ %H:%M:%S.%N`: [INFO] Invoking $$filename" > /var/tmp/post-install-$${filename}.log +exec >> /var/tmp/post-install-$${filename}.log 2>&1 + +USER=ubuntu +mkdir -p /home/$$USER/.ssh +cat >> /home/$$USER/.ssh/authorized_keys << 'EOF' +%{ for ssh_key in bastion_ssh_keys ~} +backend ${ssh_key} +%{ endfor ~} +EOF + +# Change ownership and access modes for the new directory/file +chown -R $$USER:$$USER /home/$$USER/.ssh +chmod -R go-rx /home/$$USER/.ssh diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..41dc51f --- /dev/null +++ b/variables.tf @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +variable "project_prefix" { + type = string +} + +variable "project_fqdn" { + type = string +} + +variable "project_rev_fqdn" { + type = string +} + +variable "vpc_cidr" { + type = string + default = "172.31.0.0/16" +} + +variable "ip_whitelist" { + type = list(string) + default = [] +} + +variable "config_output_path" { +} + +variable "bastion_ssh_keys" { + type = list(string) +} + +variable "kubectl_assume_role" { + default = "" +} + +variable "spot_price" { + default = "" +} + +variable "key_name" { + type = string +} + +variable "bastion_instance_size" { + type = string + default = "t3.nano" +} + +variable "eks_authorized_roles" { + type = list(string) + default = [] +} + +variable "instance_types" { + type = list(string) +} + +variable "worker_groups" { + type = any + default = [ + { + instance_type = "t3.large" + }, + { + instance_type = "t3.2xlarge" + } + ] +} + +variable "extra_policy_arn" { + type = string + default = "arn:aws:iam::aws:policy/AmazonS3FullAccess" +} diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..8fd41fa --- /dev/null +++ b/versions.tf @@ -0,0 +1,10 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/vpc.tf b/vpc.tf new file mode 100644 index 0000000..007f4be --- /dev/null +++ b/vpc.tf @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2019 Risk Focus, Inc. - All Rights Reserved + * You may use, distribute and modify this code under the + * terms of the Apache License Version 2.0. + * http://www.apache.org/licenses + */ + + +locals { + type_public = { + "type" = "public" + } + type_private = { + "type" = "private" + } +} + +//noinspection MissingModule +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = ">=2.15.0" + cidr = var.vpc_cidr + name = local.vpc_name + tags = merge( + local.vpc_tags, + { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + }, + ) + public_subnet_tags = local.type_public + private_subnet_tags = local.type_private + public_route_table_tags = local.type_public + private_route_table_tags = local.type_private + + public_subnets = [ + cidrsubnet(cidrsubnet(var.vpc_cidr, 2, 2), 4, 0), + cidrsubnet(cidrsubnet(var.vpc_cidr, 2, 2), 4, 1), + ] + + # TODO: use data to get AZS + azs = [ + "${local.aws_region}a", + "${local.aws_region}b", + ] + + private_subnets = [ + cidrsubnet(var.vpc_cidr, 2, 0), + cidrsubnet(var.vpc_cidr, 2, 1), + ] + + enable_dns_hostnames = true + enable_dns_support = true + enable_nat_gateway = true + single_nat_gateway = true +} + +resource "aws_security_group" "whitelist" { + name = "${var.project_prefix}-eks-whilelist" + description = "Set of whitelisted IPs for ${var.project_prefix} + GitHub hooks" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 0 + protocol = -1 + cidr_blocks = var.ip_whitelist + } + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = concat(local.github_meta_hooks, local.atlassian_inbound) + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = concat(local.github_meta_hooks, local.atlassian_inbound) + } +}