Merge pull request #8 from kolayne-IU-assignments/lab8

Lab8

app_go/Dockerfile

@@ -6,6 +6,7 @@ COPY go.mod *.go /usr/src/app/
 # No dependencies yet
 #RUN go mod download && go mod verify
 
+RUN ["env", "CGO_ENABLED=0", "go", "get"]
 RUN ["env", "CGO_ENABLED=0", "go", "build", "-o", "catfact_webapp", "."]
 
 

app_go/go.mod

@@ -1,3 +1,15 @@
 module catfact_webapp
 
 go 1.21.6
+
+require github.com/prometheus/client_golang v1.19.0
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+)

app_go/go.sum

@@ -0,0 +1,16 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=

app_go/main.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-func handler(w http.ResponseWriter, r *http.Request) {
+func index(w http.ResponseWriter, r *http.Request) {
 	fact, err := catFact()
 	if err == nil {
         w.WriteHeader(http.StatusOK)
@@ -17,9 +22,55 @@ func handler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+
+var (
+	reqCnt = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "go_requests_count",
+		Help: "Number of HTTP requests",
+	})
+
+	reqHandleTime = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name: "go_request_handle_time",
+		Help: "Time to handle a request",
+	})
+)
+
+func noteTimeMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		defer func() {
+			var dtSec float64 = time.Since(start).Seconds()
+			reqCnt.Inc()
+			reqHandleTime.Observe(dtSec)
+		}()
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+
 func main() {
-	http.HandleFunc("/", handler)
+	businessLogic := http.NewServeMux()
+	businessLogic.Handle("/", asHandler(index))
+	// Note: keeping /metrics under middleware too for consistency with app_py
+	businessLogic.Handle("/metrics", promhttp.Handler())
+
+	wrapped := noteTimeMiddleware(businessLogic)
+
 	hostPort := "0.0.0.0:5000"
 	_, _ = fmt.Println("Listening on http://" + hostPort)
-	log.Fatal(http.ListenAndServe(hostPort, nil))
+	log.Fatal(http.ListenAndServe(hostPort, wrapped))
+}
+
+
+type dummyHandler struct {
+	handlerFunc func (http.ResponseWriter, *http.Request)
+}
+
+func (h dummyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.handlerFunc(w, r)
+}
+
+func asHandler(handlerFunc func (http.ResponseWriter, *http.Request)) dummyHandler {
+	return dummyHandler{handlerFunc: handlerFunc}
 }

app_go/main_test.go

@@ -13,7 +13,7 @@ import (
 func TestFactLoads(t *testing.T) {
     w := httptest.NewRecorder()
 
-    handler(w, nil)
+    index(w, nil)
     resp := w.Result()
 
     if resp.StatusCode != http.StatusOK {

app_python/moscow_time/__init__.py

@@ -1,14 +1,31 @@
 import datetime
+from time import monotonic
 
-from flask import Flask
+from flask import Flask, request, Response
 import requests
+import prometheus_client
 
 from .cache import cache_for
 
 
 app = Flask(__name__)
 
 
+REQUEST_COUNT = prometheus_client.Counter('py_requests_count', 'Number of HTTP requests')
+REQUEST_HANDLE_TIME = prometheus_client.Histogram('py_request_handle_time', 'Time to handle a request')
+
+@app.before_request
+def note_request_start_time():
+    request.start_time = monotonic()
+
+@app.after_request
+def update_prometheus(response):
+    handle_time = monotonic() - request.start_time
+    REQUEST_COUNT.inc()
+    REQUEST_HANDLE_TIME.observe(handle_time)
+    return response
+
+
 # In case of high load, to avoid frequent requests, cache results for
 # one second
 @cache_for(1000)
@@ -30,3 +47,7 @@ def index():
     time = get_time()
     return f"In MSK it's {time.hour}:{time.minute}:{time.second}. " \
         "Have you brushed your teeth today yet?"
+
+@app.route('/metrics')
+def prometheus_metrics():
+    return Response(prometheus_client.generate_latest(), mimetype='text/plain')

app_python/requirements.txt

@@ -1,2 +1,3 @@
 Flask~=3.0.0
 requests~=2.31.0
+prometheus_client~=0.20.0

monitoring/METRICS.md

@@ -0,0 +1,26 @@
+# Prometheus metrics
+
+## Screenshots
+
+![Prometheus targets](pics/prometheus_targets.png)
+
+![Grafana promtail dashboard](pics/grafana_promtail_dashboard.png)
+
+![Grafana prometheus dashboard (impoarted)](pics/grafana_prometheus_dashboard_(imported).png)
+
+## Enhancements
+
+Grafana, Loki, and Promtail each have the RAM limit of 100MiB, Prometheus's memory is limited
+to 50MiB, limits for app_py and app_go are 30MiB and 20MiB respectively.
+
+## Metrics from web apps
+
+The Python app exports metrics corresponding to the web app (requests count, request handle
+time), as well as metrics related to python runtime exposed by the `promteheus_client`, for instance:
+
+![Prometheus, python web_app](pics/prometheus_app_py_sample.png)
+
+The Go app exports metrics corresponding to the web app (requests count, request handle time),
+as well as metrics related to go runtime exposed by the prometheus client, for instance:
+
+![Prometheus, go web_app](pics/prometheus_app_go_sample.png)

monitoring/docker-compose.yml

@@ -1,16 +1,34 @@
 networks:
   loki:
+  prometheus:
+
+volumes:
+  grafana-storage:
 
 services:
   app_py:
-    image: kolay0ne/app_py:lab6
+    image: kolay0ne/app_py:lab8
     ports:
       - "5000:5000"
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 30M}}
+    networks:
+      - prometheus
 
   app_go:
-    image: kolay0ne/app_go:lab6
+    image: kolay0ne/app_go:lab8
     ports:
       - "5500:5000"
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 20M}}
+    networks:
+      - prometheus
 
   loki:
     image: grafana/loki:2.9.2
@@ -19,6 +37,12 @@ services:
     #command: -config.file=/etc/loki/local-config.yaml
     networks:
       - loki
+      - prometheus
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 100M}}
 
   promtail:
     image: grafana/promtail:2.9.2
@@ -28,16 +52,43 @@ services:
     command: -config.file=/etc/promtail/config.yml
     networks:
       - loki
+      - prometheus
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 100M}}
 
   grafana:
     image: grafana/grafana:10.0.12
     ports:
       - "3000:3000"
     volumes:
+      - grafana-storage:/var/lib/grafana
       - ./grafana_datasources.yml:/etc/grafana/provisioning/datasources/ds.yaml
     environment:
       - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
     networks:
       - loki
+      - prometheus
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 100M}}
+
+  prometheus:
+    image: prom/prometheus:v2.51.0
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus_config.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - prometheus
+    logging:
+      options:
+        max-size: 5m
+    deploy:
+      resources: {limits: {memory: 50M}}

monitoring/grafana_datasources.yml

@@ -1,11 +1,35 @@
 apiVersion: 1
+
 datasources:
-- name: Loki
-  type: loki
-  access: proxy
-  orgId: 1
-  url: http://loki:3100
-  basicAuth: false
-  isDefault: true
-  version: 1
-  editable: false
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: true
+    version: 1
+    editable: false
+
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    jsonData:
+      httpMethod: POST
+      manageAlerts: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.51.0
+      cacheLevel: 'High'
+      disableRecordingRules: false
+      incrementalQueryOverlapWindow: 10m
+      examplarTraceIdDestinations:
+        # Field with internal link pointing to data source in Grafana.
+        # datasourceUid value can be anything, but it should be unique across all defined data source uids.
+        - datasourceUid: my_jaeger_uid
+          name: traceID
+
+        # Field with external link.
+        - name: traceID
+          url: 'http://grafana:3000/explore?orgId=1&left=%5B%22now-1h%22,%22now%22,%22Jaeger%22,%7B%22query%22:%22$${__value.raw}%22%7D%5D'
+    editable: true

monitoring/prometheus_config.yml

@@ -0,0 +1,26 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: loki
+    static_configs:
+      - targets: ['loki:3100']
+
+  - job_name: grafana
+    static_configs:
+      - targets: ['grafana:3000']
+
+  - job_name: app_py
+    static_configs:
+      - targets: ['app_py:5000']
+
+  - job_name: app_go
+    static_configs:
+      - targets: ['app_go:5000']

monitoring/promtail_config.yml

@@ -3,9 +3,6 @@ server:
   grpc_listen_port: 0
   log_level: "warn"
 
-positions:
-  filename: /var/lib/promtail/positions/positions.yaml
-
 client:
   url: http://loki:3100/api/prom/push