Skip to content

Commit

Permalink
Fix/loki (#63)
Browse files Browse the repository at this point in the history
fix(loki): upgrade loki to version 2.9.1
fix(loki): remove aws variables from loki job
fix(loki): implement tsdb storage
feat(loki): implement tsdb shipper

---------

Signed-off-by: Bruce Becker <[email protected]>
  • Loading branch information
brucellino authored Oct 14, 2023
1 parent 1fd5fb5 commit 28d3b56
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 35 deletions.
16 changes: 8 additions & 8 deletions loki/loki-local.nomad
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

variable "loki_version" {
type = string
default = "v2.7.1"
default = "v2.9.1"
}

job "loki" {
Expand Down Expand Up @@ -113,13 +113,13 @@ ingester:
schema_config:
configs:
- from: 2020-05-15
store: boltdb
object_store: filesystem
schema: v11
index:
prefix: index_
period: 168h
- from: 2020-05-15
store: boltdb
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb:
Expand Down
17 changes: 14 additions & 3 deletions loki/loki-local.yml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,31 @@ ingester:
schema_config:
configs:
- from: 2020-05-15
store: boltdb
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 168h

storage_config:
boltdb:
directory: /tmp/loki/index
boltdb-shipper:
active_index_directory: /data/loki/index
build_per_tenant_index: true
cache_location: /data/botdb-cache
directory: /data/loki/index
shared_store: cloudflare
tsdb_shipper:
active_index_directory: /data/tsdb-index
cache_location: /data/tsdb-cache
shared_store: cloudflare

filesystem:
directory: /tmp/loki/chunks

query_scheduler:
max_outstanding_requests_per_tenant: 32768

limits_config:
enforce_metric_name: false
reject_old_samples: true
Expand Down
22 changes: 11 additions & 11 deletions loki/loki.nomad
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
variable "access_key" {
type = string
}
// variable "access_key" {
// type = string
// }

variable "secret_key" {
type = string
}
// variable "secret_key" {
// type = string
// }

variable "loki_version" {
type = string
default = "v2.7.1"
default = "v2.9.1"
}

job "loki" {
Expand Down Expand Up @@ -71,10 +71,10 @@ job "loki" {
change_mode = "signal"
change_signal = "SIGUSR1"
}
env {
access_key = var.access_key
secret_key = var.secret_key
}
// env {
// access_key = var.access_key
// secret_key = var.secret_key
// }
config {
command = "loki"
args = [
Expand Down
44 changes: 31 additions & 13 deletions loki/loki.yml.tpl
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{{ with secret "hashiatho.me-v2/loki_logs_bucket" }}
auth_enabled: false

server:
Expand All @@ -24,29 +25,46 @@ ingester:
chunk_retain_period: 30s
schema_config:
configs:
- from: 2020-01-01
- from: "2020-01-01"
store: aws
object_store: s3
schema: v11
index:
prefix: loki_
- from: "2023-10-15"
index:
period: 24h
prefix: index_
object_store: aws
schema: v12
store: tsdb

storage_config:
aws:
region: ams3
endpoint: {{ with secret "hashiatho.me-v2/loki_logs_bucket" }}"https://{{ .Data.data.account_id }}{{ end }}
bucketnames: {{ key "jobs/loki/logs_bucket" }}
access_key_id: {{ with secret "hashiatho.me-v2/loki_logs_bucket" }}{{ .Data.data.access_key_id }}{{ end }}
secret_access_key: {{ with secret "hashiatho.me-v2/loki_logs_bucket" }}{{ .Data.data.secret_access_key }}{{ end }}
region: auto
endpoint: "https://{{ .Data.data.account_id }}r2.cloudflarestorage.com"
bucketnames: hah-logs
access_key_id: {{ .Data.data.access_key_id }}
secret_access_key: {{ .Data.data.secret_access_key }}
s3forcepathstyle: true
insecure: false
sse_encryption: false
http_config:
idle_conn_timeout: 90s
insecure_skip_verify: false
dynamodb:
dynamodb_url: inmemory:///index
dynamodb_url: inmemory:///loki
boltdb_shipper:
active_index_directory: /loki/index
cache_location: /loki/index_cache
shared_store: s3
ruler:
active_index_directory: /data/index
cache_location: /data/boltdb-cache
shared_store: aws
build_per_tenant_index: true
tsdb_shipper:
active_index_directory: /data/tsdb-index
cache_location: /data/tsdb-cache
shared_store: aws
{{/* ruler:
storage:
s3:
bucketnames: {{ key "jobs/loki/logs_bucket" }}
aws:
bucketnames: {{ key "jobs/loki/logs_bucket" }} */}}
{{ end }}

0 comments on commit 28d3b56

Please sign in to comment.