feat: working secret consumption job (#22)

feat: working secret consumption job chore: update version of promtail to v2.7 chore: comment out grpc checks feat(loki): add simple loki jobf inmem feat(loki): working version of loki standalone chore: initial commit of ansible step-up job ci: add initial job for nomad job validation ci: initial commit of vault step ci: split validation workflow into two jobs ci: add a condition for running validation only on nomad changes fix(weather): file is already staged ci: fix nomad path fix(loki): fix name of the journal unit ci: add step to plan the job ci: fix validate nomad step to safely create bin dir ci: add pre-commit autoupdate fix(loki): fix flush period fix(prometheus): query consul on localhost chore: add ansible job fix(grafana): tune resources and version for grafana fix(loki): add trailing slash to kv store for loki collectors fix(prometheus): add relabel configs for consul job lookup fix: fix name of the nomad token policy used in secret demo job Signed-off-by: Bruce Becker <[email protected]> --------- Signed-off-by: Bruce Becker <[email protected]>
brucellino · Apr 9, 2023 · 2c7a7cd · 2c7a7cd
1 parent 673be55
commit 2c7a7cd
Show file tree

Hide file tree

Showing 14 changed files with 543 additions and 262 deletions.
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -0,0 +1,37 @@
+name: Nomad Validate
+on:
+  pull_request:
+    branches:
+     - main
+    paths:
+      - '**.nomad'
+
+jobs:
+  tailscale:
+    env:
+      NOMAD_ADDR: ${{ secrets.NOMAD_ADDR }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Setup Tailscale
+        uses: tailscale/github-action@main
+        with:
+          authkey: ${{ secrets.TAILSCALE_AUTHKEY }}
+  nomad-plan:
+    needs: tailscale
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Get Vault
+        run: |
+          mkdir -p bin ; curl -fSL https://releases.hashicorp.com/vault/1.12.3/vault_1.12.3_linux_amd64.zip | gunzip -> bin/vault
+      - name: Get token
+        run: chmod u+x bin/vault ; bin/vault -version
+      - name: Get Nomad
+        run: |
+          mkdir -p bin ; curl -fSL https://releases.hashicorp.com/nomad/1.4.4/nomad_1.4.4_linux_amd64.zip | gunzip -> bin/nomad
+      - name: Checkout change
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Plan the job
+        run: chmod u+x bin/nomad ; bin/nomad plan ansible.nomad
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -26,3 +26,14 @@ repos:
     rev: v1.26.3
     hooks:
       - id: tfsec-system
+
+  - repo: local
+    hooks:
+      - id: packer-fmt
+        name: Packer Format
+        language: system
+        types:
+          - hcl
+        entry: packer fmt
+ci:
+  autoupdate_branch: main
diff --git a/ansible.nomad b/ansible.nomad
@@ -0,0 +1,36 @@
+# Job to add Ansible to all nodes, in order to allow them to
+# configure themselves
+# This job should install Ansible in a system-wide place.
+job "ansible" {
+  type        = "sysbatch"
+  datacenters = ["dc1"]
+  name        = "Ansible"
+
+  periodic {
+    cron    = "@daily"
+    enabled = true
+  }
+
+  group "nodes" {
+    count = 1
+
+    task "step-up" {
+      template {
+        change_mode = "noop"
+        destination = "local/install-ansible.sh"
+        perms       = "0777"
+
+        data = <<EOT
+#!/bin/env bash
+python3 -m pip install ansible
+EOT
+      }
+
+      driver = "raw_exec"
+
+      config {
+        command = "local/install-ansible.sh"
+      }
+    }
+  }
+}
diff --git a/consul-snapshot.nomad b/consul-snapshot.nomad
@@ -7,26 +7,26 @@ job "consul-backup" {
   group "data" {
     count = 1
     network {}
-    volume "scratch" {
-      type = "host"
-      source = "scratch"
-      read_only = false
-    }
+    // volume "scratch" {
+    //   type = "host"
+    //   source = "scratch"
+    //   read_only = false
+    // }
     task "get-terraform" {
       driver = "exec"
-      lifecycle {
+    lifecycle {
         hook = "prestart"
         sidecar = false
       }
       config {
         command = "bash"
-        args = ["-c", "curl https://releases.hashicorp.com/terraform/1.3.4/terraform_1.3.4_linux_arm64.zip | gunzip ->terraform ; chmod u+x terraform"]
-      }
-      volume_mount {
-        volume = "scratch"
-        destination = "/volume"
-        read_only = false
+        args = ["-c", "curl https://r1eleases.hashicorp.com/terraform/1.3.4/terraform_1.3.4_linux_arm64.zip | gunzip ->terraform ; chmod u+x terraform"]
       }
+      // volume_mount {
+      //   volume = "scratch"
+      //   destination = "/volume"
+      //   read_only = false
+      // }
     }
     task "check-consul" {
       driver = "exec"

diff --git a/grafana.nomad b/grafana.nomad
@@ -1,6 +1,6 @@
 variable "grafana_version" {
   type = string
-  default = "8.5.0"
+  default = "9.4.7"
   description = "Grafana version"
 }
 
@@ -25,7 +25,7 @@ job "dashboard" {
   # select machines with more than 4GB of RAM
   constraint {
     attribute = "${attr.memory.totalbytes}"
-    value     = "1GB"
+    value     = "500MB"
     operator  = ">="
   }
   update {
@@ -62,8 +62,8 @@ job "dashboard" {
         type = "tcp"
         port = "mysql_server"
         name = "mysql_alive"
-        interval = "20s"
-        timeout = "2s"
+        interval = "30s"
+        timeout = "5s"
       }
     }
 
@@ -105,16 +105,16 @@ job "dashboard" {
 
     service {
       name = "grafana"
-      tags = ["monitoring", "dashboard"]
+      tags = ["monitoring", "dashboard", "urlprefix-/grafana"]
       port = "grafana_server"
 
       check {
         port = "grafana_server"
         name     = "grafana-api"
         path     = "/api/health"
         type     = "http"
-        interval = "20s"
-        timeout  = "5s"
+        interval = "10m"
+        timeout  = "10s"
       }
     }
 
@@ -141,7 +141,7 @@ job "dashboard" {
       }
       resources {
         cpu    = 1000
-        memory = 1024
+        memory = 512
       }
 
       config {
@@ -182,7 +182,7 @@ reporting_enabled = false
 external_enabled = false
 [security]
 admin_user = admin
-admin_password = "admin"
+admin_password = "admin"  #pragma: allowlist secret
 disable_gravatar = true
 [dashboards]
 versions_to_keep = 10

diff --git a/loki.nomad b/loki.nomad
@@ -0,0 +1,85 @@
+variable "loki_version" {
+  type = string
+  default = "v2.7.5"
+}
+
+job "loki" {
+  datacenters = ["dc1"]
+  type = "service"
+  name = "loki"
+
+  meta {
+    auto-backup = true
+    backup-schedule = "@hourly"
+    backup-target-db = "postgres"
+  }
+  update {
+    max_parallel = 2
+    health_check = "checks"
+    min_healthy_time = "5s"
+    healthy_deadline = "300s"
+    progress_deadline = "10m"
+    auto_revert = true
+    auto_promote = true
+    canary = 1
+  }
+  priority = 80
+  group "log-server" {
+    count = 1
+
+    network {
+      port "http" {
+        static = 3100
+      }
+      port "grpc" {
+        static = 9096
+      }
+    }
+    service {
+      name = "loki-http-server"
+      tags = ["urlprefix-/loki strip=/loki"]
+      port = "http"
+      on_update = "require_healthy"
+
+      check {
+        name = "loki_ready"
+        type = "http"
+        path = "/ready"
+        port = "http"
+        interval = "10s"
+        timeout = "3s"
+      }
+    }
+
+    service {
+      name = "loki-grpc"
+      port = "grpc"
+    }
+
+    task "server" {
+      driver = "exec"
+      config {
+        command = "loki"
+        args = [
+          "-config.file=local/loki.yml"
+        ]
+      }
+      resources {
+        cpu = 128
+        memory = 200
+      }
+      template {
+        data = file("loki.yml.tpl")
+        destination = "local/loki.yml"
+        change_mode = "restart"
+      }
+      artifact {
+        source = "https://github.com/grafana/loki/releases/download/${var.loki_version}/loki-linux-${attr.cpu.arch}.zip"
+        options { # checksum depends on the cpu arch
+        }
+        destination = "local/loki"
+        mode = "file"
+      }
+    }
+  }
+}
diff --git a/loki.yml.tpl b/loki.yml.tpl
@@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: {{ env "NOMAD_PORT_http" }}
+  grpc_listen_port: {{ env "NOMAD_PORT_grpc" }}
+  register_instrumentation: true
+  http_server_read_timeout: "40s"
+  http_server_write_timeout: "50s"
+
+{{/* distributor:
+  ring:
+    kvstore:
+      store: consul
+      prefix: loki/collectors */}}
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: consul
+        prefix: loki/collectors
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  flush_op_timeout: 20m
+schema_config:
+  configs:
+    - from: 2022-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: loki_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: local/index
+    cache_location: local/index_cache
+  filesystem:
+    directory: local/index
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+compactor:
+  working_directory: local/data/compactor
+  shared_store: filesystem
+  compaction_interval: 5m