Update AWS terraform plans to re-use modules across each other

.gitignore

@@ -17,6 +17,7 @@ inventory/terraform.py
 tests/spec/*/*runtime_spec.rb
 contrib-plugins/*
 vault-security.yaml
-terraform/*/.terraform
+terraform/**/.terraform
 roles/coreos_bootstrap
 roles/coreos_timezone
+terraform/aws-public/etcd_discovery_url.txt

bootstrap/aws/config-default.sh → bootstrap/aws/private-cloud/config-default.sh

@@ -11,7 +11,7 @@ export TF_VAR_key_file=${TF_VAR_key_file:-$HOME/.ssh/apollo_aws_rsa}
 export TF_VAR_key_name=${TF_VAR_key_name:-apollo}
 
 # Overrides default folder in Terraform.py inventory.
-export TF_VAR_STATE_ROOT="${APOLLO_ROOT}/terraform/aws"
+export TF_VAR_STATE_ROOT="${APOLLO_ROOT}/terraform/${APOLLO_PROVIDER}"
 
 export ANSIBLE_SSH_ARGS="-F ${APOLLO_ROOT}/terraform/${APOLLO_PROVIDER}/ssh.config -q"
 

bootstrap/aws-public/config-default.sh → bootstrap/aws/public-cloud/config-default.sh

@@ -6,19 +6,12 @@ export ATLAS_INFRASTRUCTURE=${ATLAS_INFRASTRUCTURE:-capgemini/apollo}
 
 export TF_VAR_access_key=${TF_VAR_access_key:?"Need to set TF_VAR_access_key non-empty"}
 export TF_VAR_secret_key=${TF_VAR_secret_key:?"Need to set TF_VAR_secret_key non-empty"}
-export TF_VAR_key_file=${TF_VAR_key_file:-$HOME/.ssh/apollo_aws_rsa}
-export TF_VAR_key_name=${TF_VAR_key_name:-deployer}
 
 # Overrides default folder in Terraform.py inventory.
-export TF_VAR_STATE_ROOT="${APOLLO_ROOT}/terraform/aws-public"
+export TF_VAR_STATE_ROOT="${APOLLO_ROOT}/terraform/${APOLLO_PROVIDER}"
 
 export ANSIBLE_SSH_ARGS="-F ${APOLLO_ROOT}/terraform/${APOLLO_PROVIDER}/ssh.config -q"
 
 export TF_VAR_region=${TF_VAR_region:-eu-west-1}
-export TF_VAR_master_instance_type=${TF_VAR_master_instance_type:-m3.medium}
-export TF_VAR_slave_instance_type=${TF_VAR_slave_instance_type:-m3.medium}
-export TF_VAR_slaves=${TF_VAR_slaves:-1}
-export TF_VAR_availability_zones=${TF_VAR_availability_zones:-'eu-west-1a,eu-west-1b,eu-west-1c'}
-
 export APOLLO_consul_dc=${APOLLO_consul_dc:-$TF_VAR_region}
 export APOLLO_mesos_cluster_name=${APOLLO_mesos_cluster_name:-$TF_VAR_region}

terraform/aws/bastion-server.tf → terraform/aws/private-cloud/bastion-server.tf

@@ -9,18 +9,17 @@ module "ami_bastion" {
 resource "aws_instance" "bastion" {
   ami               = "${module.ami_bastion.ami_id}"
   instance_type     = "${var.bastion_instance_type}"
-  subnet_id         = "${aws_subnet.public.id}"
-  security_groups   = ["${aws_security_group.default.id}", "${aws_security_group.bastion.id}"]
-  depends_on        = ["aws_internet_gateway.public", "aws_key_pair.deployer"]
-  key_name          = "${aws_key_pair.deployer.key_name}"
+  subnet_id         = "${module.vpc.public_subnets}"
+  security_groups   = ["${module.sg-default.security_group_id}", "${aws_security_group.bastion.id}"]
+  key_name          = "${module.aws-keypair.keypair_name}"
   source_dest_check = false
   tags = {
     Name = "apollo-mesos-bastion"
     role = "bastion"
   }
   connection {
     user        = "ubuntu"
-    private_key = "${var.ssh_private_key}"
+    private_key = "${var.private_key_file}"
   }
   provisioner "remote-exec" {
     inline = [

terraform/aws/private-cloud/main.tf

@@ -0,0 +1,85 @@
+variable "access_key" {}
+variable "secret_key" {}
+variable "public_key_file" { default = "~/.ssh/id_rsa_aws.pub" }
+variable "private_key_file" { default = "~/.ssh/id_rsa_aws.pem" }
+variable "region" { default = "eu-west-1" }
+variable "availability_zones" { default = "eu-west-1a,eu-west-1b,eu-west-1c" }
+variable "vpc_cidr_block" { default = "10.0.0.0/16" }
+variable "coreos_channel" { default = "stable" }
+variable "etcd_discovery_url_file" { default = "etcd_discovery_url.txt" }
+variable "masters" { default = "3" }
+variable "master_instance_type" { default = "m3.medium" }
+variable "slaves" { default = "1" }
+variable "slave_instance_type" { default = "m3.medium" }
+variable "bastion_instance_type" { default = "t2.micro" }
+variable "docker_version" { default = "1.9.1-0~trusty" }
+
+provider "aws" {
+  access_key = "${var.access_key}"
+  secret_key = "${var.secret_key}"
+  region     = "${var.region}"
+}
+
+module "vpc" {
+  source          = "github.com/terraform-community-modules/tf_aws_vpc"
+
+  name            = "default"
+
+  cidr            = "${var.vpc_cidr_block}"
+  private_subnets = "10.0.1.0/24,10.0.2.0/24,10.0.3.0/24"
+  public_subnets  = "10.0.101.0/24,10.0.102.0/24,10.0.103.0/24"
+
+  azs             = "${var.availability_zones}"
+}
+
+# ssh keypair for instances
+module "aws-keypair" {
+  source = "../keypair"
+
+  public_key_filename = "${var.public_key_file}"
+}
+
+# security group to allow all traffic in and out of the instances in the VPC
+module "sg-default" {
+  source = "../sg-all-traffic"
+
+  vpc_id = "${module.vpc.vpc_id}"
+}
+
+module "elb" {
+  source = "../elb"
+
+  security_groups = "${module.sg-default.security_group_id}"
+  instances       = "${join(\",\", aws_instance.mesos-slave.*.id)}"
+  subnets         = "${module.vpc.public_subnets}"
+}
+
+# Generate an etcd URL for the cluster
+resource "template_file" "etcd_discovery_url" {
+  template = "/dev/null"
+  provisioner "local-exec" {
+    command = "curl https://discovery.etcd.io/new?size=${var.masters + var.slaves} > ${var.etcd_discovery_url_file}"
+  }
+  # This will regenerate the discovery URL if the cluster size changes
+  vars {
+    size = "${var.masters + var.slaves}"
+  }
+}
+
+# outputs
+output "bastion.ip" {
+  value = "${aws_eip.bastion.public_ip}"
+}
+output "master.1.ip" {
+  value = "${aws_instance.mesos-master.0.private_ip}"
+}
+output "master_ips" {
+  value = "${join(",", aws_instance.mesos-master.*.private_ip)}"
+}
+output "slave_ips" {
+  value = "${join(",", aws_instance.mesos-slave.*.private_ip)}"
+}
+/*
+output "elb.hostname" {
+  value = "${module.elb.elb_dns_name}"
+}*/

terraform/aws/mesos-masters.tf → terraform/aws/private-cloud/mesos-masters.tf

@@ -23,11 +23,11 @@ resource "aws_instance" "mesos-master" {
   instance_type     = "${var.master_instance_type}"
   ami               = "${module.master_ami.ami_id}"
   count             = "${var.masters}"
-  key_name          = "${aws_key_pair.deployer.key_name}"
+  key_name          = "${module.aws-keypair.keypair_name}"
   source_dest_check = false
   subnet_id         = "${element(aws_subnet.private.*.id, count.index)}"
-  security_groups   = ["${aws_security_group.default.id}"]
-  depends_on        = ["aws_instance.bastion", "aws_internet_gateway.public"]
+  security_groups   = ["${module.sg-default.security_group_id}"]
+  depends_on        = ["aws_instance.bastion"]
   user_data         = "${template_file.master_cloud_init.rendered}"
   tags = {
     Name = "apollo-mesos-master-${count.index}"

terraform/aws/mesos-slaves.tf → terraform/aws/private-cloud/mesos-slaves.tf

@@ -23,45 +23,14 @@ resource "aws_instance" "mesos-slave" {
   instance_type     = "${var.slave_instance_type}"
   ami               = "${module.slave_ami.ami_id}"
   count             = "${var.slaves}"
-  key_name          = "${aws_key_pair.deployer.key_name}"
+  key_name          = "${module.aws-keypair.keypair_name}"
   source_dest_check = false
   subnet_id         = "${element(aws_subnet.private.*.id, count.index)}"
-  security_groups   = ["${aws_security_group.default.id}"]
-  depends_on        = ["aws_instance.bastion", "aws_internet_gateway.public", "aws_instance.mesos-master"]
+  security_groups   = ["${module.sg-default.security_group_id}"]
+  depends_on        = ["aws_instance.bastion", "aws_instance.mesos-master"]
   user_data         = "${template_file.master_cloud_init.rendered}"
   tags = {
     Name = "apollo-mesos-slave-${count.index}"
     role = "mesos_slaves"
   }
 }
-
-# Load balancer
-resource "aws_elb" "app" {
-  name            = "apollo-mesos-elb"
-  subnets         = ["${aws_subnet.public.*.id}"]
-  security_groups = ["${aws_security_group.default.id}", "${aws_security_group.web.id}"]
-
-  listener {
-    instance_port     = 80
-    instance_protocol = "http"
-    lb_port           = 80
-    lb_protocol       = "http"
-  }
-
-  # traefik health check
-  health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    timeout             = 3
-    target              = "HTTP:8888/health"
-    interval            = 30
-  }
-
-  instances                 = ["${aws_instance.mesos-slave.*.id}"]
-  cross_zone_load_balancing = true
-}
-
-resource "aws_proxy_protocol_policy" "http" {
-  load_balancer  = "${aws_elb.app.name}"
-  instance_ports = ["80"]
-}

terraform/aws/private-subnet.tf → terraform/aws/private-cloud/private-subnet.tf

@@ -1,6 +1,6 @@
 # Private subnet
 resource "aws_subnet" "private" {
-  vpc_id                  = "${aws_vpc.default.id}"
+  vpc_id                  = "${module.vpc.vpc_id}"
   count                   = "${length(split(",", var.availability_zones))}"
   availability_zone       = "${element(split(",", var.availability_zones), count.index)}"
   cidr_block              = "10.0.${count.index+1}.0/24"
@@ -12,7 +12,7 @@ resource "aws_subnet" "private" {
 }
 
 resource "aws_route_table" "private" {
-  vpc_id = "${aws_vpc.default.id}"
+  vpc_id = "${module.vpc.vpc_id}"
   route {
     cidr_block  = "0.0.0.0/0"
     instance_id = "${aws_instance.bastion.id}"

terraform/aws/security_groups.tf → terraform/aws/private-cloud/security_groups.tf

@@ -1,31 +1,7 @@
-resource "aws_security_group" "default" {
-  name        = "default-apollo-mesos"
-  description = "Default security group that allows inbound and outbound traffic from all instances in the VPC"
-  vpc_id      = "${aws_vpc.default.id}"
-
-  ingress {
-    from_port   = "0"
-    to_port     = "0"
-    protocol    = "-1"
-    self        = true
-  }
-
-  egress {
-    from_port   = "0"
-    to_port     = "0"
-    protocol    = "-1"
-    self        = true
-  }
-
-  tags {
-    Name = "apollo-mesos-default-vpc"
-  }
-}
-
 resource "aws_security_group" "bastion" {
   name        = "bastion-apollo-mesos"
   description = "Security group for bastion instances that allows SSH and VPN traffic from internet"
-  vpc_id      = "${aws_vpc.default.id}"
+  vpc_id      = "${module.vpc.vpc_id}"
 
   ingress {
     from_port   = 22
@@ -63,7 +39,7 @@ resource "aws_security_group" "bastion" {
 resource "aws_security_group" "web" {
   name = "web-apollo-mesos"
   description = "Security group that allows web traffic from the internet"
-  vpc_id = "${aws_vpc.default.id}"
+  vpc_id      = "${module.vpc.vpc_id}"
 
   ingress {
     from_port   = 80

terraform/aws/public-cloud/etcd_discovery_url.txt

@@ -0,0 +1 @@
+https://discovery.etcd.io/fafb16ac572cae5895c6523e7b3bd640

terraform/aws-public/main.tf → terraform/aws/public-cloud/main.tf

@@ -1,3 +1,16 @@
+variable "access_key" {}
+variable "secret_key" {}
+variable "public_key_file" { default = "~/.ssh/id_rsa_aws.pub" }
+variable "region" { default = "eu-west-1" }
+variable "availability_zones" { default = "eu-west-1a,eu-west-1b,eu-west-1c" }
+variable "coreos_channel" { default = "stable" }
+variable "etcd_discovery_url_file" { default = "etcd_discovery_url.txt" }
+variable "masters" { default = "3" }
+variable "master_instance_type" { default = "m3.medium" }
+variable "slaves" { default = "1" }
+variable "slave_instance_type" { default = "m3.medium" }
+variable "vpc_cidr_block" { default = "10.0.0.0/16" }
+
 provider "aws" {
   access_key = "${var.access_key}"
   secret_key = "${var.secret_key}"
@@ -15,7 +28,7 @@ resource "aws_vpc" "default" {
 
 # ssh keypair for instances
 module "aws-keypair" {
-  source = "./keypair"
+  source = "../keypair"
 
   public_key_filename = "${var.public_key_file}"
 }
@@ -41,13 +54,13 @@ module "public_subnet" {
 
 # security group to allow all traffic in and out of the instances
 module "sg-default" {
-  source = "./sg-all-traffic"
+  source = "../sg-all-traffic"
 
   vpc_id = "${aws_vpc.default.id}"
 }
 
 module "elb" {
-  source = "./elb"
+  source = "../elb"
 
   security_groups = "${module.sg-default.security_group_id}"
   instances       = "${join(\",\", aws_instance.mesos-slave.*.id)}"