From 08ae614164467dce277f6d9d840e82a3031c8448 Mon Sep 17 00:00:00 2001 From: Bruce Becker Date: Sat, 29 Jun 2024 10:41:48 +0200 Subject: [PATCH 1/4] feat: add job for hashi-at-home consul and nomad secrets Signed-off-by: Bruce Becker --- consul-snapshot.nomad | 64 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/consul-snapshot.nomad b/consul-snapshot.nomad index a62e46a..8df2ef2 100644 --- a/consul-snapshot.nomad +++ b/consul-snapshot.nomad @@ -1,38 +1,48 @@ +# A job to retrieve a consul token from Vault. job "consul-backup" { datacenters = ["dc1"] - type = "batch" - periodic { - cron = "1-59/5 * * * * *" - } + type = "batch" + // periodic { + // crons = ["1-59/15 * * * * *"] + // prohibit_overlap = false + // } group "data" { count = 1 network {} - // volume "scratch" { - // type = "host" - // source = "scratch" - // read_only = false - // } - task "get-terraform" { - driver = "exec" - lifecycle { - hook = "prestart" - sidecar = false - } - config { - command = "bash" - args = ["-c", "curl https://r1eleases.hashicorp.com/terraform/1.3.4/terraform_1.3.4_linux_arm64.zip | gunzip ->terraform ; chmod u+x terraform"] - } - // volume_mount { - // volume = "scratch" - // destination = "/volume" - // read_only = false - // } + vault { + env = true } + restart { + attempts = 0 + } + task "check-consul" { - driver = "exec" + # Get a vault token so that we can read consul creds + template { + data = <<-EOH +#!/bin/env bash +echo "Hi there! I'm a dufus" +# Lookup vault token +curl -H "X-Vault-Token: ${VAULT_TOKEN}" \ + -X GET \ + http://active.vault.service.consul:8200/v1/auth/token/lookup-self > output.json +cat output.json | jq +sleep 120 + + EOH + destination = "local/start.sh" + perms = "777" + } + template { + data = <<-EOH +{{ with secret "hashi_at_home/creds/cluster-role" }}{{ .Data.token }}{{ end }} + EOH + destination = "local/consul_token" + perms = "400" + } + driver = "raw_exec" config { - command = "bash" - args = ["-c", "consul -version"] + command = "local/start.sh" } } } From e10cd713f6b1b564d8f3926fc9ba9f5bfb551145 Mon Sep 17 00:00:00 2001 From: Bruce Becker Date: Sun, 30 Jun 2024 11:20:04 +0200 Subject: [PATCH 2/4] fix: add secrets for CONSUL and VAULT to env with template Signed-off-by: Bruce Becker --- consul-snapshot.nomad | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/consul-snapshot.nomad b/consul-snapshot.nomad index 8df2ef2..a9f0fa5 100644 --- a/consul-snapshot.nomad +++ b/consul-snapshot.nomad @@ -2,10 +2,10 @@ job "consul-backup" { datacenters = ["dc1"] type = "batch" - // periodic { - // crons = ["1-59/15 * * * * *"] - // prohibit_overlap = false - // } + periodic { + crons = ["1-59/15 * * * * *"] + prohibit_overlap = false + } group "data" { count = 1 network {} @@ -15,32 +15,41 @@ job "consul-backup" { restart { attempts = 0 } - task "check-consul" { # Get a vault token so that we can read consul creds template { data = <<-EOH #!/bin/env bash +env +source ${NOMAD_SECRETS_DIR}/env +env echo "Hi there! I'm a dufus" # Lookup vault token -curl -H "X-Vault-Token: ${VAULT_TOKEN}" \ - -X GET \ - http://active.vault.service.consul:8200/v1/auth/token/lookup-self > output.json -cat output.json | jq -sleep 120 - +curl -v -X GET -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://localhost:8500/v1/snapshot > ${NOMAD_ALLOC_DIR}/data/snapshot +# Get the playbook +curl -X GET \ + -H "Accept: applicaton/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-Github-Api-Version: 2022-11-28" \ + https://api.github.com/repos/brucellino/personal-automation/contents/playbooks/backup-state.yml > playbook.yml +ls -lht EOH destination = "local/start.sh" perms = "777" } template { data = <<-EOH -{{ with secret "hashi_at_home/creds/cluster-role" }}{{ .Data.token }}{{ end }} +CONSUL_HTTP_TOKEN="{{ with secret "hashi_at_home/creds/cluster-role" }}{{ .Data.token }}{{ end }}" +GITHUB_TOKEN="{{ with secret "/github_personal_tokens/token" "repositories=personal-automation" +"installation_id=44668070"}}{{ .Data.token }}{{ end }}" +NOMAD_ADDR={{ with service "http.nomad" }}{{ with index . 0 }}http://{{ .Address }}:{{ .Port }}{{ end }}{{ end }} +VAULT_ADDR={{ range service "vault" }}http://{{ .Address }}:{{ .Port }}{{- end }} EOH - destination = "local/consul_token" + destination = "${NOMAD_SECRETS_DIR}/env" perms = "400" + env = true } - driver = "raw_exec" + driver = "exec" config { command = "local/start.sh" } From d8137f92aaa0cf6c763c6d24876fbd8f9b3b37f0 Mon Sep 17 00:00:00 2001 From: Bruce Becker Date: Sun, 30 Jun 2024 21:30:24 +0200 Subject: [PATCH 3/4] feat: add backup job for H@H services Signed-off-by: Bruce Becker --- consul-snapshot.nomad => hah-snapshot.nomad | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename consul-snapshot.nomad => hah-snapshot.nomad (100%) diff --git a/consul-snapshot.nomad b/hah-snapshot.nomad similarity index 100% rename from consul-snapshot.nomad rename to hah-snapshot.nomad From 546b1f82e0184467ee4b3ef491e6b71a1cd8f46a Mon Sep 17 00:00:00 2001 From: Bruce Becker Date: Sun, 30 Jun 2024 21:34:07 +0200 Subject: [PATCH 4/4] chore: change execution cron to 15 past every 6th hour Signed-off-by: Bruce Becker --- hah-snapshot.nomad | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/hah-snapshot.nomad b/hah-snapshot.nomad index a9f0fa5..0f09217 100644 --- a/hah-snapshot.nomad +++ b/hah-snapshot.nomad @@ -1,12 +1,12 @@ # A job to retrieve a consul token from Vault. -job "consul-backup" { +job "hah-state-backup" { datacenters = ["dc1"] type = "batch" periodic { crons = ["1-59/15 * * * * *"] prohibit_overlap = false } - group "data" { + group "all" { count = 1 network {} vault { @@ -15,24 +15,39 @@ job "consul-backup" { restart { attempts = 0 } - task "check-consul" { + task "snapshot" { # Get a vault token so that we can read consul creds template { data = <<-EOH #!/bin/env bash -env +PATH=${HOME}/.local/bin:${PATH} source ${NOMAD_SECRETS_DIR}/env -env -echo "Hi there! I'm a dufus" -# Lookup vault token -curl -v -X GET -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://localhost:8500/v1/snapshot > ${NOMAD_ALLOC_DIR}/data/snapshot +echo Nomad addr: ${NOMAD_ADDR} +ecoh +# Generate Consul Snapshot +curl -v \ + -X GET \ + -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" \ + http://localhost:8500/v1/snapshot \ + > ${NOMAD_ALLOC_DIR}/data/consul_$(date --iso-8601=date).snap + +# Generate Nomad snapshot +curl -v \ + -X GET \ + -H "X-Nomad-Token: ${NOMAD_TOKEN}" \ + ${NOMAD_ADDR}/v1/operator/snapshot \ + > ${NOMAD_ALLOC_DIR}/data/nomad_$(date --iso-8601=date).snap + # Get the playbook curl -X GET \ -H "Accept: applicaton/vnd.github+json" \ -H "Authorization: Bearer ${GITHUB_TOKEN}" \ -H "X-Github-Api-Version: 2022-11-28" \ - https://api.github.com/repos/brucellino/personal-automation/contents/playbooks/backup-state.yml > playbook.yml -ls -lht + https://api.github.com/repos/brucellino/personal-automation/contents/playbooks/backup-state.yml \ + | jq -r .content \ + | base64 -d > playbook.yml +pip install --user ansible boto3 botocore +ansible-playbook -c local -i localhost, playbook.yml EOH destination = "local/start.sh" perms = "777" @@ -40,9 +55,11 @@ ls -lht template { data = <<-EOH CONSUL_HTTP_TOKEN="{{ with secret "hashi_at_home/creds/cluster-role" }}{{ .Data.token }}{{ end }}" +CONSUL_HTTP_ADDR=http://localhost:8500 GITHUB_TOKEN="{{ with secret "/github_personal_tokens/token" "repositories=personal-automation" "installation_id=44668070"}}{{ .Data.token }}{{ end }}" NOMAD_ADDR={{ with service "http.nomad" }}{{ with index . 0 }}http://{{ .Address }}:{{ .Port }}{{ end }}{{ end }} +NOMAD_TOKEN="{{ with secret "nomad/creds/mgmt" }}{{ .Data.secret_id }}{{ end }}" VAULT_ADDR={{ range service "vault" }}http://{{ .Address }}:{{ .Port }}{{- end }} EOH destination = "${NOMAD_SECRETS_DIR}/env"