diff --git a/AkkaDotNet.LargeNetworkTests.sln b/AkkaDotNet.LargeNetworkTests.sln index 13e20af..c18eb6d 100644 --- a/AkkaDotNet.LargeNetworkTests.sln +++ b/AkkaDotNet.LargeNetworkTests.sln @@ -15,8 +15,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "frontend", "frontend", "{39 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AkkaDotNet.FrontEnd", "src\frontend\AkkaDotNet.FrontEnd\AkkaDotNet.FrontEnd.csproj", "{696FA3DA-FE78-43B8-B24C-942662B51D2E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AkkaDotNet.FrontEnd.Tests", "src\frontend\AkkaDotNet.FrontEnd.Tests\AkkaDotNet.FrontEnd.Tests.csproj", "{3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{6E8AB83C-075B-4333-8DFC-17FE9EFDDBA0}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AkkaDotNet.Infrastructure", "src\shared\AkkaDotNet.Infrastructure\AkkaDotNet.Infrastructure.csproj", "{98CB4416-9594-425E-990D-771170F92D36}" @@ -27,6 +25,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "backend", "backend", "{BE2E EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AkkaDotNet.Messages", "src\shared\AkkaDotNet.Messages\AkkaDotNet.Messages.csproj", "{8F96042B-A435-4789-A92A-67B9C433E293}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTracing.FrontEnd", "src\frontend\OpenTracing.FrontEnd\OpenTracing.FrontEnd.csproj", "{3B7DBF90-D6BB-4684-A88B-4066A7003D02}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTracing.Infrastructure", "src\shared\OpenTracing.Infrastructure\OpenTracing.Infrastructure.csproj", "{FA6B1637-0CB9-405E-BD71-609CA3FE0D64}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTracing.BackEnd", "src\backend\OpenTracing.BackEnd\OpenTracing.BackEnd.csproj", "{B4F1F5AA-6164-489B-8F2C-E8886E949393}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -39,10 +43,6 @@ Global {696FA3DA-FE78-43B8-B24C-942662B51D2E}.Debug|Any CPU.Build.0 = Debug|Any CPU {696FA3DA-FE78-43B8-B24C-942662B51D2E}.Release|Any CPU.ActiveCfg = Release|Any CPU {696FA3DA-FE78-43B8-B24C-942662B51D2E}.Release|Any CPU.Build.0 = Release|Any CPU - {3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93}.Release|Any CPU.Build.0 = Release|Any CPU {98CB4416-9594-425E-990D-771170F92D36}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {98CB4416-9594-425E-990D-771170F92D36}.Debug|Any CPU.Build.0 = Debug|Any CPU {98CB4416-9594-425E-990D-771170F92D36}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -55,6 +55,18 @@ Global {8F96042B-A435-4789-A92A-67B9C433E293}.Debug|Any CPU.Build.0 = Debug|Any CPU {8F96042B-A435-4789-A92A-67B9C433E293}.Release|Any CPU.ActiveCfg = Release|Any CPU {8F96042B-A435-4789-A92A-67B9C433E293}.Release|Any CPU.Build.0 = Release|Any CPU + {3B7DBF90-D6BB-4684-A88B-4066A7003D02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3B7DBF90-D6BB-4684-A88B-4066A7003D02}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3B7DBF90-D6BB-4684-A88B-4066A7003D02}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3B7DBF90-D6BB-4684-A88B-4066A7003D02}.Release|Any CPU.Build.0 = Release|Any CPU + {FA6B1637-0CB9-405E-BD71-609CA3FE0D64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA6B1637-0CB9-405E-BD71-609CA3FE0D64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA6B1637-0CB9-405E-BD71-609CA3FE0D64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA6B1637-0CB9-405E-BD71-609CA3FE0D64}.Release|Any CPU.Build.0 = Release|Any CPU + {B4F1F5AA-6164-489B-8F2C-E8886E949393}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B4F1F5AA-6164-489B-8F2C-E8886E949393}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B4F1F5AA-6164-489B-8F2C-E8886E949393}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B4F1F5AA-6164-489B-8F2C-E8886E949393}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -64,9 +76,11 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {696FA3DA-FE78-43B8-B24C-942662B51D2E} = {397C6273-19B5-4DA8-8D1F-09185B0D8A02} - {3358DBCC-E7A8-4CAF-A49C-599CEA2B1A93} = {397C6273-19B5-4DA8-8D1F-09185B0D8A02} {98CB4416-9594-425E-990D-771170F92D36} = {6E8AB83C-075B-4333-8DFC-17FE9EFDDBA0} {044D0104-F02F-439F-A55C-5FB378D51AF5} = {BE2EB7C9-9D6A-4F8A-8226-B21428C5AC7C} {8F96042B-A435-4789-A92A-67B9C433E293} = {6E8AB83C-075B-4333-8DFC-17FE9EFDDBA0} + {3B7DBF90-D6BB-4684-A88B-4066A7003D02} = {397C6273-19B5-4DA8-8D1F-09185B0D8A02} + {FA6B1637-0CB9-405E-BD71-609CA3FE0D64} = {6E8AB83C-075B-4333-8DFC-17FE9EFDDBA0} + {B4F1F5AA-6164-489B-8F2C-E8886E949393} = {BE2EB7C9-9D6A-4F8A-8226-B21428C5AC7C} EndGlobalSection EndGlobal diff --git a/k8s/local_otrace/README.md b/k8s/local_otrace/README.md new file mode 100644 index 0000000..d63a627 --- /dev/null +++ b/k8s/local_otrace/README.md @@ -0,0 +1,3 @@ +# Local K8s Testing + +This is designed for deployment on a local development machine, not a full-blown AKS cluster. \ No newline at end of file diff --git a/k8s/local_otrace/deployAll.cmd b/k8s/local_otrace/deployAll.cmd new file mode 100644 index 0000000..1ec061c --- /dev/null +++ b/k8s/local_otrace/deployAll.cmd @@ -0,0 +1,33 @@ +@echo off +REM deploys all Kubernetes services to their staging environment + +set namespace=akkastress +set location=%~dp0/environment + +echo "Deploying K8s resources from [%location%] into namespace [%namespace%]" + +echo "Creating Namespaces..." +kubectl create ns "%namespace%" + +echo "Using namespace [%namespace%] going forward..." + +echo "Creating configurations from YAML files in [%location%/configs]" +for %%f in (%location%/configs/*.yaml) do ( + echo "Deploying %%~nxf" + kubectl apply -f "%location%/configs/%%~nxf" -n "%namespace%" +) + +echo "Creating environment-specific services from YAML files in [%location%]" +for %%f in (%location%/*.yaml) do ( + echo "Deploying %%~nxf" + kubectl apply -f "%location%/%%~nxf" -n "%namespace%" +) + +echo "Creating all services..." +for %%f in (%~dp0/services/*.yaml) do ( + echo "Deploying %%~nxf" + kubectl apply -f "%~dp0/services/%%~nxf" -n "%namespace%" +) + +echo "All services started... Printing K8s output.." +kubectl get all -n "%namespace%" \ No newline at end of file diff --git a/k8s/local_otrace/destroyAll.cmd b/k8s/local_otrace/destroyAll.cmd new file mode 100644 index 0000000..2af4aef --- /dev/null +++ b/k8s/local_otrace/destroyAll.cmd @@ -0,0 +1,4 @@ +@echo off +REM destroys all K8s services in "phobos-web" namespace + +kubectl delete ns akkastress \ No newline at end of file diff --git a/k8s/local_otrace/environment/configs/configs.yaml b/k8s/local_otrace/environment/configs/configs.yaml new file mode 100644 index 0000000..a4ee649 --- /dev/null +++ b/k8s/local_otrace/environment/configs/configs.yaml @@ -0,0 +1,23 @@ +# staging settings +kind: ConfigMap +apiVersion: v1 +metadata: + name: stress-configs +data: + # Configuration values can be set as key-value properties + ASPNETCORE_ENVIRONMENT: Production + + # standard Akka.NET ports for all services + StressOptions__AkkaClusterOptions__ManagementPort: "9221" + StressOptions__AkkaClusterOptions__ManagementPort: "9228" + StressOptions__AkkaClusterOptions__UseKubernetesLease: "true" + + # configure Akka.Management K8s service discovery + StressOptions__AkkaClusterOptions__UseKubernetesDiscovery: "true" + StressOptions__AkkaClusterOptions__KubernetesDiscoveryOptions__PodNamespace: "akkastress" + StressOptions__AkkaClusterOptions__KubernetesDiscoveryOptions__PodLabelSelector: "akkacluster=stress" + + # configure Serilog + StressOptions__SerilogOptions__EnableSeq: "true" + StressOptions__SerilogOptions__SeqHost: "seq" + StressOptions__SerilogOptions__SeqPort: "8988" \ No newline at end of file diff --git a/k8s/local_otrace/environment/configs/k8s-lease-crd.yaml b/k8s/local_otrace/environment/configs/k8s-lease-crd.yaml new file mode 100644 index 0000000..6af289c --- /dev/null +++ b/k8s/local_otrace/environment/configs/k8s-lease-crd.yaml @@ -0,0 +1,54 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + # name must match the spec fields below, and be in the form: . + name: leases.akka.io +spec: + group: akka.io + versions: + - name: v1 + storage: true + served: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + owner: + type: string + version: + type: string + time: + type: integer + scope: Namespaced + names: + # kind is normally the CamelCased singular type. Your resource manifests use this. + kind: Lease + listKind: LeaseList + # singular name to be used as an alias on the CLI and for display + singular: lease + # plural name to be used in the URL: /apis/// + plural: leases +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: lease-access +rules: + - apiGroups: ["akka.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "list"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: lease-access +subjects: + - kind: User + name: system:serviceaccount:akkastress:default +roleRef: + kind: Role + name: lease-access + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/k8s/local_otrace/environment/configs/rbac-reader.yaml b/k8s/local_otrace/environment/configs/rbac-reader.yaml new file mode 100644 index 0000000..3c8538b --- /dev/null +++ b/k8s/local_otrace/environment/configs/rbac-reader.yaml @@ -0,0 +1,29 @@ +# +# Create a role, `pod-reader`, that can list pods and +# bind the default service account in the namespace +# that the binding is deployed to to that role. +# + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: pod-reader +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: read-pods +subjects: + # Uses the default service account. + # Consider creating a dedicated service account to run your + # Akka Cluster services and binding the role to that one. +- kind: ServiceAccount + name: default +roleRef: + kind: Role + name: pod-reader + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/k8s/local_otrace/environment/grafana-configmap.yaml b/k8s/local_otrace/environment/grafana-configmap.yaml new file mode 100644 index 0000000..4e2faf3 --- /dev/null +++ b/k8s/local_otrace/environment/grafana-configmap.yaml @@ -0,0 +1,3866 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: grafana + name: grafana-dash-provider +data: + providers.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: false + options: + path: /var/lib/grafana/dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: grafana + name: grafana-dashs +data: + k8s-dashboard.json: | + { + "__inputs": [{ + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }], + "__requires": [{ + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.0" + }, { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }], + "id": null, + "title": "Kubernetes Pod Resources", + "description": "Shows resource usage of Kubernetes pods.", + "tags": [ + "kubernetes" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [{ + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (container_memory_working_set_bytes{id=\"/\",instance=~\"^$instance$\"}) / sum (machine_memory_bytes{instance=~\"^$instance$\"}) * 100", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 2 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Memory Working Set", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\",instance=~\"^$instance$\"}[1m])) / sum (machine_cpu_cores{instance=~\"^$instance$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Cpu Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{id=\"/\",instance=~\"^$instance$\"}) / sum(container_fs_limit_bytes{id=\"/\",instance=~\"^$instance$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Filesystem Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_memory_working_set_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (machine_memory_bytes{instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",instance=~\"^$instance$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "timeShift": null, + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (machine_cpu_cores{instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (container_fs_limit_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "receive", + "metric": "network", + "refId": "A", + "step": 240 + }, { + "expr": "- sum(rate(container_network_transmit_bytes_total{instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "transmit", + "metric": "network", + "refId": "B", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Network", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "Bps", + "label": "transmit / receive", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }], + "showTitle": true, + "title": "all pods" + }, { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Cpu Usage", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 33, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Memory Working Set", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "bytes", + "label": "used", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }} < in", + "metric": "network", + "refId": "A", + "step": 240 + }, { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }} > out", + "metric": "network", + "refId": "B", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Network", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "Bps", + "label": "transmit / receive", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 34, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "bytes", + "label": "used", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }], + "showTitle": true, + "title": "each pod" + }], + "time": { + "from": "now-3d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [{ + "allValue": ".*", + "current": {}, + "datasource": "prometheus", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "type": "query" + }, { + "current": {}, + "datasource": "prometheus", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(namespace)", + "refresh": 1, + "regex": "", + "type": "query" + }] + }, + "annotations": { + "list": [] + }, + "refresh": false, + "schemaVersion": 12, + "version": 8, + "links": [], + "gnetId": 737 + } + akkadotnet-dashboard.json: | + { + "__inputs": [{ + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }], + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "System-wide Akka.NET metrics used for profiling Akka.NET performance. Gathered via Phobos and OpenTelemetry.", + "editable": true, + "gnetId": 15637, + "graphTooltip": 0, + "id": 4, + "iteration": 1645797823962, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 23, + "panels": [], + "title": "Cluster Status", + "type": "row" + }, + { + "datasource": "prometheus", + "description": "Total number of active Akka.Cluster Members.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 0, + "y": 1 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"Up\"}))", + "instant": true, + "interval": "", + "legendFormat": "Up", + "refId": "A" + }, + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"WeaklyUp\"}))", + "instant": true, + "interval": "", + "legendFormat": "WeaklyUp", + "refId": "B" + }, + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"Joining\"}))", + "instant": true, + "interval": "", + "legendFormat": "Joining", + "refId": "C" + }, + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"Leaving\"}))", + "instant": true, + "interval": "", + "legendFormat": "Leaving", + "refId": "D" + }, + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"Exiting\"}))", + "instant": true, + "interval": "", + "legendFormat": "Exiting", + "refId": "E" + }, + { + "expr": "ceil(avg(akka_cluster_members_members{status=\"Down\"}))", + "instant": true, + "interval": "", + "legendFormat": "Down", + "refId": "F" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Active Cluster Size", + "transformations": [], + "type": "stat" + }, + { + "datasource": "prometheus", + "description": "All nodes in the cluster that are detected to be unreachable.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "ceil(avg(akka_cluster_reachable_members{status=\"False\"}))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unreachable Nodes", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Tracks all nodes by their membership status and reachability.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "ceil(avg by(status)(akka_cluster_members_members))", + "interval": "", + "legendFormat": "{{status}}", + "refId": "A" + }, + { + "expr": "ceil(avg(akka_cluster_reachable_members{status=\"False\"}))", + "instant": false, + "interval": "", + "legendFormat": "Unreachable", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Nodes by Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "panels": [], + "title": "Logging and Error Rates", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Total aggregate logging activity per-second for each node in an Akka.NET cluster.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "error", + "color": "#F2495C", + "fill": 1 + }, + { + "alias": "warning", + "color": "#FA6400", + "fill": 2 + }, + { + "alias": "info", + "color": "#73BF69" + }, + { + "alias": "debug", + "color": "#C8F2C2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(level)(increase(akka_logs_events{cluster_role=~\"$role\",akka_address=~\"$address\"}[1m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{level}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Logging Activity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "prometheus", + "description": "Groups all exceptions detected in Akka.NET logs by type.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 10, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(cluster_role,akka_address,exceptiontype)(ceil(increase(akka_logs_events{cluster_role=~\"$role\",akka_address=~\"$address\",exceptiontype=~\".+\"}[5m])))", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Exceptions by Type (over 5m)", + "transform": "table", + "type": "table-old" + }, + { + "cacheTimeout": null, + "datasource": "prometheus", + "description": "Total errors logged by Akka.NET over past 5 minutes, cumulative for all roles / cluster / exception types.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 12, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "sum(increase(akka_logs_events{level=\"error\",cluster_role=~\"$role\",akka_address=~\"$address\"}[5m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total Errors Logged (5m)", + "type": "stat" + }, + { + "collapsed": false, + "datasource": "prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 2, + "panels": [], + "repeat": "role", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "title": "Actor Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "The full set of Akka.NET actors running on all nodes for all roles in a cluster.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(cluster_role,akka_address)(akka_actor_created_starts{cluster_role=~\"$role\",akka_address=~\"$address\"}) - sum by(cluster_role,akka_address)(akka_actor_stopped_stops{cluster_role=~\"$role\",akka_address=~\"$address\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total Actors [{{cluster_role}} ({{akka_address}})]", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Akka.NET Actors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "datasource": "prometheus", + "description": "Cumulative Akka.NET messaging throughput across all selected nodes and roles.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 16, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.7", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "targets": [ + { + "expr": "sum(irate(akka_messages_recv_msgs{cluster_role=~\"$role\",akka_address=~\"$address\"}[1m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total Actor Throughput (msg/s)", + "type": "stat" + }, + { + "columns": [], + "datasource": "prometheus", + "description": "Top 10 actor starts / stops by type.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 14, + "links": [], + "pageSize": null, + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "pattern": "Time", + "type": "date" + }, + { + "alias": "Created", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Terminated", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10,sum by(actortype)(akka_actor_created_starts{cluster_role=~\"$role\",akka_address=~\"$address\"}))", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{actortype}} created", + "refId": "A" + }, + { + "expr": "topk(10,sum by(actortype)(akka_actor_stopped_stops{cluster_role=~\"$role\",akka_address=~\"$address\"}))", + "instant": true, + "interval": "", + "legendFormat": "{{actortype}} terminated", + "refId": "B" + } + ], + "title": "Live Actors by Type (Top 10)", + "transform": "table", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "type": "table-old" + }, + { + "datasource": "prometheus", + "description": "Top 10 actor type / exception type pairs, based on the number of crashes per second observed over a 30 minute timespan.", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 20, + "options": { + "showHeader": true + }, + "pluginVersion": "7.3.7", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "targets": [ + { + "expr": "topk(10,sum by(actortype, exceptiontype)(rate(akka_actor_restarts_restarts[30m])))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{actortype}},{{exceptiontype}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Actor Crashes by Actor / Exception (Top 10)", + "type": "table" + }, + { + "datasource": "prometheus", + "description": "Akka.NET message throughput by message type.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 18, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.3.7", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "targets": [ + { + "expr": "topk(10,sum by(messagetype)(rate(akka_messages_recv_msgs{cluster_role=~\"$role\",akka_address=~\"$address\"}[5m])))", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{messagetype}}", + "refId": "A" + }, + { + "expr": "sum by(messagetype)(irate(akka_messages_recv_msgs{cluster_role=~\"$role\",akka_address=~\"$address\"}[1m]))", + "hide": true, + "instant": false, + "interval": "", + "legendFormat": "{{messagetype}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Akka.NET Message Throughput by Type (Top 10)", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": "prometheus", + "description": "Akka.NET message throughput by message type.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 21, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.3.7", + "scopedVars": { + "role": { + "selected": false, + "text": "console", + "value": "console" + } + }, + "targets": [ + { + "expr": "topk(10,sum by(messagetype)(increase(akka_messages_recv_msgs{cluster_role=~\"$role\",akka_address=~\"$address\"}[5m])))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{messagetype}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Akka.NET Message Totals by Type (Top 10 over 5 minutes)", + "transformations": [], + "type": "bargauge" + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "akka.net", + "phobos" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(cluster_role)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Role", + "multi": true, + "name": "role", + "options": [], + "query": "label_values(cluster_role)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(akka_address)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "address", + "multi": true, + "name": "address", + "options": [], + "query": "label_values(akka_address)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "datasource": null, + "error": null, + "filters": [], + "hide": 0, + "label": "", + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Akka.NET Cluster + Phobos 2.x Metrics (Prometheus Data Source)", + "uid": "fxcTFhb7z", + "version": 1 + } + akkadotnet-latency-dashboard.json: | + { + "__inputs": [{ + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }], + "annotations": { + "list": [ + { + "builtIn": 1, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Message-processing latency dashboard for Akka.NET Clusters using Phobos 2.0 and OpenTelemetry.", + "editable": true, + "gnetId": 15638, + "graphTooltip": 0, + "id": 3, + "iteration": 1645798543780, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 7, + "panels": [], + "title": "[$msgtype] Cumulative Latencies", + "type": "row" + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "prometheus", + "description": "Heatmap of milliseconds of latency by $msgtype", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "legend": { + "show": false + }, + "maxDataPoints": 25, + "pluginVersion": "7.3.7", + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(increase(akka_messages_latency_ms_bucket{messagetype=~\"$msgtype\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency Over Time [$msgtype ]", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "datasource": "prometheus", + "description": "Heatmap of milliseconds of latency by $msgtype", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 9, + "maxDataPoints": 25, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "sum(akka_messages_latency_ms_bucket{messagetype=~\"$msgtype\"}) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency Distribution [$msgtype ]", + "type": "bargauge" + }, + { + "collapsed": false, + "datasource": "prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "[$msgtype] Latencies When Processed by [$actortype]", + "type": "row" + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "prometheus", + "description": "Heatmap of milliseconds of latency by $actortype / $msgtype", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 2, + "legend": { + "show": false + }, + "maxDataPoints": 25, + "pluginVersion": "7.3.7", + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(increase(akka_messages_latency_ms_bucket{messagetype=~\"$msgtype\",actortype=~\"$actortype\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency Over Time [$actortype / $msgtype ]", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "datasource": "prometheus", + "description": "Heatmap of milliseconds of latency by $actortype / $msgtype", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 3, + "maxDataPoints": 25, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.3.7", + "targets": [ + { + "expr": "sum(akka_messages_latency_ms_bucket{messagetype=~\"$msgtype\",actortype=~\"$actortype\"}) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency Distribution [$actortype / $msgtype ]", + "type": "bargauge" + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "int", + "value": "int" + }, + "datasource": "prometheus", + "definition": "label_values(messagetype)", + "error": null, + "hide": 0, + "includeAll": false, + "label": "msgtype", + "multi": false, + "name": "msgtype", + "options": [], + "query": "label_values(messagetype)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "Petabridge.Cmd.Remote.Actors.GlobalStatusActor", + "value": "Petabridge.Cmd.Remote.Actors.GlobalStatusActor" + }, + "datasource": "prometheus", + "definition": "label_values(actortype)", + "error": null, + "hide": 0, + "includeAll": false, + "label": "actortype", + "multi": false, + "name": "actortype", + "options": [], + "query": "label_values(actortype)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Akka.NET Cluster + Phobos 2.x Message Latency Metrics (Prometheus Data Source)", + "uid": "-MOG0hx7z", + "version": 1 + } + dotnet-monitor-dashboard.json: | + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.2.6" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1649985042094, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [], + "title": ".NET ThreadPool", + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_threadpool_completed_items_count{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "completed items {{instance}}", + "refId": "A" + } + ], + "title": ".NET Threadpool Completed Items", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_threadpool_thread_count{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "thread_count {{instance}}", + "refId": "A" + } + ], + "title": ".NET Threadpool Thread Count", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_threadpool_queue_length{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "queue_length {{instance}}", + "refId": "A" + } + ], + "title": ".NET Threadpool Queue Length", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 4, + "panels": [], + "title": "CPU Utilization", + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_cpu_usage_ratio{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "CPU Usage {{instance}}", + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 16, + "panels": [], + "title": "Memory and GC", + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_gen_0_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "Gen 0 Size {{instance}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "systemruntime_gen_1_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "Gen 1 Size {{instance}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "systemruntime_gen_2_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "Gen 2 Size {{instance}}", + "refId": "D" + }, + { + "exemplar": true, + "expr": "systemruntime_loh_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "LOH Size {{instance}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "systemruntime_poh_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "POH Size {{instance}}", + "refId": "E" + } + ], + "title": "Generation Sizes", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "systemruntime_gc_fragmentation_ratio{instance=\"host.docker.internal:52323\", job=\"memoryleak\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Fragmentation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_gc_fragmentation_ratio{app=~\"$app\",instance=~\"$instance\"}/100", + "interval": "", + "legendFormat": "Total Fragmentation {{instance}}", + "refId": "A" + } + ], + "title": "GC Fragmentation", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 16, + "y": 27 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_time_in_gc_ratio{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "% Time in GC {{instance}}", + "refId": "A" + } + ], + "title": "GC", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_alloc_rate_bytes{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Alloc rate {{instance}}", + "refId": "A" + } + ], + "title": "Allocations", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "systemruntime_gc_heap_size_bytes{app=~\"$app\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "GC Heap Size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "systemruntime_working_set_bytes{app=~\"$app\",instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "Working Set", + "refId": "B" + } + ], + "title": "Overall Sizes", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 32, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "definition": "label_values(app)", + "description": "Kubernetes Application Name (app:{variable})", + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "app", + "options": [], + "query": { + "query": "label_values(app)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "definition": "label_values(instance)", + "description": "The pod instance of each service", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "dotnet-monitor Kubernetes Dashboard", + "uid": "FF-1G8U7z", + "version": 1 + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: grafana + name: grafana-datasources-provider +data: + providers.yaml: | + apiVersion: 1 + datasources: + - name: 'prometheus' + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus-ip-service:9090 + isDefault: true + editable: false \ No newline at end of file diff --git a/k8s/local_otrace/environment/grafana-service.yaml b/k8s/local_otrace/environment/grafana-service.yaml new file mode 100644 index 0000000..bbaf0c1 --- /dev/null +++ b/k8s/local_otrace/environment/grafana-service.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana-ip-service + annotations: + prometheus.io/scrape: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '3000' +spec: + type: LoadBalancer + selector: + component: grafana + ports: + - port: 3000 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + labels: + component: grafana +spec: + replicas: 1 + selector: + matchLabels: + component: grafana + template: + metadata: + labels: + component: grafana + spec: + containers: + - image: grafana/grafana:8.2.6 + name: grafana-core + imagePullPolicy: IfNotPresent + # env: + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + # The following env variables set up basic auth twith the default admin user and admin password. + - name: GF_AUTH_BASIC_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + # does not really work, because of template variables in exported dashboards: + # - name: GF_DASHBOARDS_JSON_ENABLED + # value: "true" + readinessProbe: + httpGet: + path: /login + port: 3000 + volumeMounts: + - name: grafana-persistent-storage + mountPath: /var/lib/grafana + - name: datasources-provider + mountPath: /etc/grafana/provisioning/datasources + - name: dashboards-volume + mountPath: /var/lib/grafana/dashboards + - name: dashboard-provider + mountPath: /etc/grafana/provisioning/dashboards + volumes: + - name: grafana-persistent-storage + emptyDir: {} + - name: datasources-provider + configMap: + name: grafana-datasources-provider + items: + - key: providers.yaml + path: providers.yaml + - name: dashboards-volume + configMap: + name: grafana-dashs + - name: dashboard-provider + configMap: + name: grafana-dash-provider + items: + - key: providers.yaml + path: providers.yaml diff --git a/k8s/local_otrace/environment/jaeger.all-in-one.yaml b/k8s/local_otrace/environment/jaeger.all-in-one.yaml new file mode 100644 index 0000000..89fd2a9 --- /dev/null +++ b/k8s/local_otrace/environment/jaeger.all-in-one.yaml @@ -0,0 +1,158 @@ +# +# Copyright 2017-2019 The Jaeger Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# + +apiVersion: v1 +kind: List +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: jaeger + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + strategy: + type: Recreate + template: + metadata: + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + annotations: + prometheus.io/scrape: "false" + prometheus.io/port: "16686" + spec: + containers: + - env: + - name: COLLECTOR_ZIPKIN_HTTP_PORT + value: "9411" + image: jaegertracing/all-in-one + name: jaeger + ports: + - containerPort: 5775 + protocol: UDP + - containerPort: 6831 + protocol: UDP + - containerPort: 6832 + protocol: UDP + - containerPort: 5778 + protocol: TCP + - containerPort: 16686 + protocol: TCP + - containerPort: 9411 + protocol: TCP + readinessProbe: + httpGet: + path: "/" + port: 14269 + initialDelaySeconds: 5 +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-query + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: query + spec: + type: LoadBalancer + ports: + - name: query-http + port: 16686 + protocol: TCP + targetPort: 16686 + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-collector + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: collector + spec: + ports: + - name: jaeger-collector-tchannel + port: 14267 + protocol: TCP + targetPort: 14267 + - name: jaeger-collector-http + port: 14268 + protocol: TCP + targetPort: 14268 + - name: jaeger-collector-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one + type: ClusterIP +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-agent + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: agent + spec: + ports: + - name: agent-zipkin-thrift + port: 5775 + protocol: UDP + targetPort: 5775 + - name: agent-compact + port: 6831 + protocol: UDP + targetPort: 6831 + - name: agent-binary + port: 6832 + protocol: UDP + targetPort: 6832 + - name: agent-configs + port: 5778 + protocol: TCP + targetPort: 5778 + clusterIP: None + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one +- apiVersion: v1 + kind: Service + metadata: + name: zipkin + labels: + app: jaeger + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: zipkin + spec: + ports: + - name: jaeger-collector-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + clusterIP: None + selector: + app.kubernetes.io/name: jaeger + app.kubernetes.io/component: all-in-one \ No newline at end of file diff --git a/k8s/local_otrace/environment/prometheus-config.yaml b/k8s/local_otrace/environment/prometheus-config.yaml new file mode 100644 index 0000000..6a709ab --- /dev/null +++ b/k8s/local_otrace/environment/prometheus-config.yaml @@ -0,0 +1,129 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: akkastress +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: akkastress +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + name: prometheus-server-conf +data: + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" + + scrape_configs: + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - "akkastress" + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - "akkastress" + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name \ No newline at end of file diff --git a/k8s/local_otrace/environment/prometheus-service.yaml b/k8s/local_otrace/environment/prometheus-service.yaml new file mode 100644 index 0000000..b75bc84 --- /dev/null +++ b/k8s/local_otrace/environment/prometheus-service.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-ip-service + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: /metrics + prometheus.io/port: '9090' +spec: + type: LoadBalancer + selector: + app: prometheus-server + ports: + - port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-deployment +spec: + selector: + matchLabels: + app: prometheus-server + replicas: 1 + template: + metadata: + labels: + app: prometheus-server + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: prom/prometheus:latest + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1000m" + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + + - name: prometheus-storage-volume + emptyDir: {} \ No newline at end of file diff --git a/k8s/local_otrace/environment/seq-deploy-local.yaml b/k8s/local_otrace/environment/seq-deploy-local.yaml new file mode 100644 index 0000000..6dc2e9f --- /dev/null +++ b/k8s/local_otrace/environment/seq-deploy-local.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Service +metadata: + name: seq +spec: + ports: + - name: seq + port: 8988 + protocol: TCP + targetPort: 80 + selector: + app: seq + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seq + labels: + app: seq +spec: + replicas: 1 + selector: + matchLabels: + app: seq + template: + metadata: + labels: + app: seq + spec: + containers: + - name: seq + image: datalust/seq:2020.5 + env: + - name: ACCEPT_EULA + value: "Y" + ports: + - containerPort: 80 + protocol: TCP \ No newline at end of file diff --git a/k8s/local_otrace/services/local-services.yaml b/k8s/local_otrace/services/local-services.yaml new file mode 100644 index 0000000..1ab6292 --- /dev/null +++ b/k8s/local_otrace/services/local-services.yaml @@ -0,0 +1,273 @@ +apiVersion: v1 +kind: Service +metadata: + name: frontend-web + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '80' + labels: + app: frontend +spec: + type: LoadBalancer + ports: + - name: query-http + port: 1880 + protocol: TCP + targetPort: 80 + selector: + app: frontend +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-akka + labels: + app: frontend +spec: + clusterIP: None + ports: + - port: 9228 + name: management + - port: 9221 + name: akka-remote + selector: + app: frontend +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '52323' + labels: + app: backend +spec: + clusterIP: None + ports: + - port: 52323 + name: metrics + selector: + app: frontend +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: frontend + labels: + app: frontend + akkacluster: stress +spec: + serviceName: "frontend-akka" + replicas: 3 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + akkacluster: stress + spec: + volumes: + - name: diagvol + emptyDir: {} + - name: dumpsvol + emptyDir: {} + containers: + - name: frontend + image: opentracing.frontend:0.5.12 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: StressOptions__AkkaClusterOptions__Hostname + value: "$(POD_NAME).frontend-akka" + - name: DOTNET_DiagnosticPorts + value: /diag/port + envFrom: + - configMapRef: + name: stress-configs + volumeMounts: + - mountPath: /diag + name: diagvol + - mountPath: /dumps + name: dumpsvol + readinessProbe: + httpGet: + path: "/ready" + port: 80 + ports: + - containerPort: 9228 + name: management + protocol: TCP + - containerPort: 9221 + name: akka-remote + protocol: TCP + - containerPort: 80 + name: http + protocol: TCP + - name: monitor + image: mcr.microsoft.com/dotnet/monitor:6.0 + # DO NOT use the --no-auth argument for deployments in production; this argument is used for demonstration + # purposes only in this example. Please continue reading after this example for further details. + args: [ "--no-auth" ] + imagePullPolicy: Always + env: + - name: DOTNETMONITOR_DiagnosticPort__ConnectionMode + value: Listen + - name: DOTNETMONITOR_DiagnosticPort__EndpointName + value: /diag/port + - name: DOTNETMONITOR_Storage__DumpTempFolder + value: /dumps + # ALWAYS use the HTTPS form of the URL for deployments in production; the removal of HTTPS is done for + # demonstration purposes only in this example. Please continue reading after this example for further details. + - name: DOTNETMONITOR_Urls + value: http://*:52323 + ports: + - containerPort: 52323 + name: metrics + volumeMounts: + - mountPath: /diag + name: diagvol + - mountPath: /dumps + name: dumpsvol + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: backend-akka + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '80' + labels: + app: backend +spec: + clusterIP: None + ports: + - port: 9228 + name: management + - port: 9221 + name: akka-remote + selector: + app: backend +--- +apiVersion: v1 +kind: Service +metadata: + name: backend-monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + prometheus.io/port: '52323' + labels: + app: backend +spec: + clusterIP: None + ports: + - port: 52323 + name: metrics + selector: + app: backend +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: backend + labels: + app: backend + akkacluster: stress +spec: + serviceName: "backend-akka" + replicas: 3 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + akkacluster: stress + spec: + volumes: + - name: diagvol + emptyDir: {} + - name: dumpsvol + emptyDir: {} + containers: + - name: backend + image: opentracing.backend:0.5.12 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: StressOptions__AkkaClusterOptions__Hostname + value: "$(POD_NAME).backend-akka" + - name: DOTNET_DiagnosticPorts + value: /diag/port + envFrom: + - configMapRef: + name: stress-configs + volumeMounts: + - mountPath: /diag + name: diagvol + - mountPath: /dumps + name: dumpsvol + readinessProbe: + httpGet: + path: "/ready" + port: 80 + ports: + - containerPort: 9228 + name: management + protocol: TCP + - containerPort: 9221 + name: akka-remote + protocol: TCP + - containerPort: 80 + name: http + protocol: TCP + - name: monitor + image: mcr.microsoft.com/dotnet/monitor:6.0 + # DO NOT use the --no-auth argument for deployments in production; this argument is used for demonstration + # purposes only in this example. Please continue reading after this example for further details. + args: [ "--no-auth" ] + imagePullPolicy: Always + env: + - name: DOTNETMONITOR_DiagnosticPort__ConnectionMode + value: Listen + - name: DOTNETMONITOR_DiagnosticPort__EndpointName + value: /diag/port + - name: DOTNETMONITOR_Storage__DumpTempFolder + value: /dumps + # ALWAYS use the HTTPS form of the URL for deployments in production; the removal of HTTPS is done for + # demonstration purposes only in this example. Please continue reading after this example for further details. + - name: DOTNETMONITOR_Urls + value: http://*:52323 + ports: + - containerPort: 52323 + name: metrics + volumeMounts: + - mountPath: /diag + name: diagvol + - mountPath: /dumps + name: dumpsvol + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/src/backend/OpenTracing.BackEnd/Actors/ItemActor.cs b/src/backend/OpenTracing.BackEnd/Actors/ItemActor.cs new file mode 100644 index 0000000..1fe5488 --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/Actors/ItemActor.cs @@ -0,0 +1,114 @@ +using System; +using Akka.Actor; +using Akka.Cluster.Sharding; +using Akka.Cluster.Tools.PublishSubscribe; +using Akka.Event; +using Akka.Persistence; +using AkkaDotNet.Messages; +using AkkaDotNet.Messages.Commands; +using AkkaDotNet.Messages.Events; +using OpenTracing.Infrastructure; + +namespace OpenTracing.BackEnd.Actors; + +public class ItemActor : ReceivePersistentActor +{ + private int _count = 0; + private readonly bool _pubSubEnabled; + private readonly ILoggingAdapter _log = Context.GetLogger(); + + public static Props PropsFor(string itemId, bool pubSubEnabled) + { + return Props.Create(() => new ItemActor(itemId, pubSubEnabled)); + } + + public ItemActor(string persistenceId, bool pubSubEnabled) + { + PersistenceId = persistenceId; + _pubSubEnabled = pubSubEnabled; + + Recover(i => + { + _log.Info("Recovery: count was {0} - adding {1}", _count, i.Count); + _count += i.Count; + }); + + Recover(r => + { + _log.Info("Recovery: count was {0} - subtracting {1}", _count, r.Count); + _count -= r.Count; + }); + + Recover(o => + { + if (o.Snapshot is int i) + { + _count = i; + _log.Info("Recovered initial count value of [{0}]", i); + } + }); + + Command(i => + { + var evnt = new ItemAdded(PersistenceId, i.Count); + Persist(evnt, added => + { + _count += added.Count; + Sender.Tell(new CommandResponse(i, CommandResult.Ok)); + SaveSnapshotWhenAble(); + }); + }); + + Command(i => + { + var evnt = new ItemRemoved(PersistenceId, i.Count); + Persist(evnt, removed => + { + _count -= removed.Count; + Sender.Tell(new CommandResponse(i, CommandResult.Ok)); + SaveSnapshotWhenAble(); + }); + }); + + Command(s => + { + DeleteMessages(s.Metadata.SequenceNr); + DeleteSnapshots(new SnapshotSelectionCriteria(s.Metadata.SequenceNr-1)); + }); + + Command(_ => + { + Context.Parent.Tell(new Passivate(PoisonPill.Instance)); + }); + + Command(a => + { + _log.Info("Confirmed subscription to [{0}]", a.Subscribe.Topic); + }); + + Command(p => + { + Sender.Tell(p); + }); + } + + private void SaveSnapshotWhenAble() + { + if (LastSequenceNr % 25 == 0) + { + SaveSnapshot(_count); + } + } + + public override string PersistenceId { get; } + + protected override void PreStart() + { + Context.SetReceiveTimeout(TimeSpan.FromMinutes(2)); + if (_pubSubEnabled) + { + var mediator = DistributedPubSub.Get(Context.System).Mediator; + mediator.Tell(new Subscribe(ActorSystemConstants.PingTopicName, Self), Self); + } + } +} \ No newline at end of file diff --git a/src/backend/OpenTracing.BackEnd/Dockerfile b/src/backend/OpenTracing.BackEnd/Dockerfile new file mode 100644 index 0000000..683bc00 --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/Dockerfile @@ -0,0 +1,34 @@ +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base +WORKDIR /app + +# important environment variables +ENV StressOptions__AkkaClusterOptions__ManagementPort "9221" +ENV StressOptions__AkkaClusterOptions__ManagementPort "9228" +ENV StressOptions__AkkaClusterOptions__Hostname "" + +# 9110 - Petabridge.Cmd +# 9221 - Akka.Cluster +# 9228 - Akka.Management +# 80 - HTTP +EXPOSE 9110 9221 9228 80 + +# Install Petabridge.Cmd client so it can be invoked remotely via +# Docker or K8s 'exec` commands +RUN dotnet tool install --global pbm + +# RUN pbm help + +COPY ./bin/Release/net6.0/publish/ /app + +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS app +WORKDIR /app + +COPY --from=base /app /app + +# copy .NET Core global tool +COPY --from=base /root/.dotnet /root/.dotnet/ + +# Needed because https://stackoverflow.com/questions/51977474/install-dotnet-core-tool-dockerfile +ENV PATH="${PATH}:/root/.dotnet/tools" + +CMD ["dotnet", "OpenTracing.BackEnd.dll"] \ No newline at end of file diff --git a/src/backend/OpenTracing.BackEnd/OpenTracing.BackEnd.csproj b/src/backend/OpenTracing.BackEnd/OpenTracing.BackEnd.csproj new file mode 100644 index 0000000..ed3b102 --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/OpenTracing.BackEnd.csproj @@ -0,0 +1,24 @@ + + + + Exe + net6.0 + + + + + + + + + + + Always + + + + Always + + + + diff --git a/src/backend/OpenTracing.BackEnd/Program.cs b/src/backend/OpenTracing.BackEnd/Program.cs new file mode 100644 index 0000000..3cb0792 --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/Program.cs @@ -0,0 +1,123 @@ +using System; +using System.Threading.Tasks; +using Akka.Actor; +using Akka.Cluster.Hosting; +using Akka.Cluster.Sharding; +using Akka.Hosting; +using AkkaDotNet.Messages; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenTracing.BackEnd.Actors; +using OpenTracing.Infrastructure; +using OpenTracing.Infrastructure.Actors; +using OpenTracing.Infrastructure.Configuration; +using OpenTracing.Infrastructure.Logging; +using OpenTracing.Infrastructure.Sharding; +using Serilog; + +namespace OpenTracing.BackEnd; + +public static class Program +{ + public static async Task Main(string[] args) + { + var host = new WebHostBuilder() + .UseKestrel() + .ConfigureAppConfiguration(builder => + { + builder + .AddCommandLine(args) + .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", false); + }) + .ConfigureLogging(logging => + { + logging + .ClearProviders() + .AddConsole() + .AddSerilog() + .AddFilter(null, LogLevel.Warning);; + }) + .ConfigureServices( (context, services) => + { + var akkaConfiguration = context.Configuration.GetRequiredSection(nameof(StressOptions)).Get(); + + // sets up Akka.NET + services.AddAkka(ActorSystemConstants.ActorSystemName, configurationBuilder => + { + configurationBuilder.WithClusterBootstrap(akkaConfiguration, + new[] { ActorSystemConstants.BackendRole, ActorSystemConstants.DistributedPubSubRole }); + configurationBuilder.WithSerilog(akkaConfiguration.SerilogOptions); + configurationBuilder.WithReadyCheckActors(); + + if (akkaConfiguration.DistributedPubSubOptions.Enabled) + { + configurationBuilder.WithShardRegion("items", s => ItemActor.PropsFor(s, akkaConfiguration.DistributedPubSubOptions.Enabled), new ItemShardExtractor(), + new ShardOptions() + { + RememberEntities = akkaConfiguration.ShardingOptions.RememberEntities, + Role = ActorSystemConstants.BackendRole, + StateStoreMode = akkaConfiguration.ShardingOptions.UseDData ? StateStoreMode.DData : StateStoreMode.Persistence + }) + .WithItemMessagingActor(); + } + + }); + + // enables OpenTracing for ASP.NET Core + services.AddOpenTracing(o => + { + o.ConfigureAspNetCore(a => + { + a.Hosting.OperationNameResolver = context => $"{context.Request.Method} {context.Request.Path}"; + + // skip Prometheus HTTP /metrics collection from appearing in our tracing system + a.Hosting.IgnorePatterns.Add(x => x.Request.Path.StartsWithSegments(new PathString("/metrics"))); + }); + o.ConfigureGenericDiagnostics(c => { }); + }); + + // sets up Prometheus + ASP.NET Core metrics + services.ConfigureAppMetrics(); + + // sets up Jaeger tracing + services.ConfigureJaegerTracing(); + + services.AddRouting(); + services.AddControllers(); + }) + .Configure((context, app) => + { + if (context.HostingEnvironment.IsDevelopment()) + app.UseDeveloperExceptionPage(); + + app.UseRouting(); + + // enable App.Metrics routes + app.UseMetricsAllMiddleware(); + app.UseMetricsAllEndpoints(); + + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + + endpoints.MapGet("/ready", async (ActorRegistry registry) => + { + var readyCheck = registry.Get(); + var checkResult = await readyCheck.Ask(ReadyCheck.Instance, TimeSpan.FromSeconds(3)); + //if (checkResult.IsReady) + return Results.StatusCode(200); + return Results.StatusCode(500); + }); + }); + }) + .Build(); + + await host.RunAsync(); + } +} \ No newline at end of file diff --git a/src/backend/OpenTracing.BackEnd/appsettings.Development.json b/src/backend/OpenTracing.BackEnd/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/backend/OpenTracing.BackEnd/appsettings.json b/src/backend/OpenTracing.BackEnd/appsettings.json new file mode 100644 index 0000000..b03de67 --- /dev/null +++ b/src/backend/OpenTracing.BackEnd/appsettings.json @@ -0,0 +1,41 @@ +{ + "StressOptions": { + "DistributedPubSubOptions": { + "Enabled": false, + "NumTopics": 100 + }, + "ShardingOptions": { + "Enabled": true, + "UseDData": true, + "RememberEntities": false + }, + "AkkaClusterOptions": { + "Hostname": "localhost", + "Port": 9221, + "ManagementPort": 9228, + "UseKubernetesDiscovery": false, + "UseKubernetesLease": false, + "KubernetesDiscoveryOptions": { + "PodNamespace": "akkastress", + "PodLabelSelector": "akkacluster=stress" + }, + "SeedNodes": ["akka.tcp://LargeNetworkSys@localhost:9221"] + }, + "SerilogOptions": { + "EnableSeq": false + }, + "DispatcherConfig": "Defaults", + "PersistenceOptions": { + "Enabled": false + }, + }, + + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/src/frontend/OpenTracing.FrontEnd/Actors/PingerActor.cs b/src/frontend/OpenTracing.FrontEnd/Actors/PingerActor.cs new file mode 100644 index 0000000..b4d36ad --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/Actors/PingerActor.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Akka.Actor; +using Akka.Cluster.Tools.PublishSubscribe; +using Akka.Event; +using AkkaDotNet.Messages.Commands; +using OpenTracing.Infrastructure; + +namespace OpenTracing.FrontEnd.Actors; + +public class PingerActor : ReceiveActor, IWithTimers +{ + private sealed class DoPing + { + public static readonly DoPing Instance = new DoPing(); + private DoPing(){} + } + + private readonly ILoggingAdapter _log = Context.GetLogger(); + private readonly IActorRef _mediator; + private readonly ITracer _tracer; + + public PingerActor(ITracer tracer) + { + _tracer = tracer; + _mediator = DistributedPubSub.Get(Context.System).Mediator; + + BecomePinging(); + } + + private void WaitingForPing() + { + Receive(d => + { + _tracer.BuildSpan("Ping") + .IgnoreActiveSpan() + .StartActive(); + _mediator.Tell(new Publish(ActorSystemConstants.PingTopicName, Ping.Instance)); + BecomePinging(); + }); + } + + private void BecomePinging() + { + Become(() => Pinging(new HashSet())); + Context.SetReceiveTimeout(TimeSpan.FromSeconds(3)); + } + + private void Pinging(HashSet pings) + { + Receive(p => + { + pings.Add(Sender); + }); + + Receive(r => + { + _tracer.ActiveSpan.Finish(); + _log.Info("Received [{0}] pings from [{1}] nodes", pings.Count, pings.Select(c => c.Path.Address).Distinct().Count()); + Become(WaitingForPing); + Timers.StartSingleTimer("ping-timer", DoPing.Instance, TimeSpan.FromSeconds(5)); + Context.SetReceiveTimeout(null); // cancel receivetimeout + }); + } + + public ITimerScheduler Timers { get; set; } + + protected override void PreStart() + { + Timers.StartSingleTimer("ping-timer", DoPing.Instance, TimeSpan.FromSeconds(5)); + } +} \ No newline at end of file diff --git a/src/frontend/OpenTracing.FrontEnd/Dockerfile b/src/frontend/OpenTracing.FrontEnd/Dockerfile new file mode 100644 index 0000000..3ba8d7f --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/Dockerfile @@ -0,0 +1,34 @@ +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS base +WORKDIR /app + +# important environment variables +ENV StressOptions__AkkaClusterOptions__ManagementPort "9221" +ENV StressOptions__AkkaClusterOptions__ManagementPort "9228" +ENV StressOptions__AkkaClusterOptions__Hostname "" + +# 9110 - Petabridge.Cmd +# 9221 - Akka.Cluster +# 9228 - Akka.Management +# 80 - HTTP +EXPOSE 9110 9221 9228 80 + +# Install Petabridge.Cmd client so it can be invoked remotely via +# Docker or K8s 'exec` commands +RUN dotnet tool install --global pbm + +# RUN pbm help + +COPY ./bin/Release/net6.0/publish/ /app + +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS app +WORKDIR /app + +COPY --from=base /app /app + +# copy .NET Core global tool +COPY --from=base /root/.dotnet /root/.dotnet/ + +# Needed because https://stackoverflow.com/questions/51977474/install-dotnet-core-tool-dockerfile +ENV PATH="${PATH}:/root/.dotnet/tools" + +CMD ["dotnet", "OpenTracing.FrontEnd.dll"] \ No newline at end of file diff --git a/src/frontend/OpenTracing.FrontEnd/OpenTracing.FrontEnd.csproj b/src/frontend/OpenTracing.FrontEnd/OpenTracing.FrontEnd.csproj new file mode 100644 index 0000000..31843fa --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/OpenTracing.FrontEnd.csproj @@ -0,0 +1,23 @@ + + + + Exe + net6.0 + + + + + + + + + + Always + + + + Always + + + + diff --git a/src/frontend/OpenTracing.FrontEnd/Program.cs b/src/frontend/OpenTracing.FrontEnd/Program.cs new file mode 100644 index 0000000..db77b18 --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/Program.cs @@ -0,0 +1,124 @@ +using System; +using System.Threading.Tasks; +using Akka.Actor; +using Akka.Cluster.Hosting; +using Akka.Cluster.Sharding; +using Akka.Hosting; +using AkkaDotNet.Messages; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenTracing.FrontEnd.Actors; +using OpenTracing.Infrastructure; +using OpenTracing.Infrastructure.Actors; +using OpenTracing.Infrastructure.Configuration; +using OpenTracing.Infrastructure.Logging; +using OpenTracing.Infrastructure.Sharding; +using Serilog; + +namespace OpenTracing.FrontEnd; + +public static class Program +{ + public static async Task Main(string[] args) + { + var host = new WebHostBuilder() + .UseKestrel() + .ConfigureAppConfiguration(builder => + { + builder + .AddCommandLine(args) + .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", false); + }) + .ConfigureLogging(logging => + { + logging + .ClearProviders() + .AddConsole() + .AddSerilog() + .AddFilter(null, LogLevel.Warning);; + }) + .ConfigureServices( (context, services) => + { + var akkaConfiguration = context.Configuration.GetRequiredSection(nameof(StressOptions)).Get(); + + // sets up Akka.NET + services.AddAkka(ActorSystemConstants.ActorSystemName, (configurationBuilder, provider) => + { + configurationBuilder.WithClusterBootstrap(akkaConfiguration, + new[] { ActorSystemConstants.FrontendRole, ActorSystemConstants.DistributedPubSubRole }); + configurationBuilder.WithSerilog(akkaConfiguration.SerilogOptions); + configurationBuilder.WithReadyCheckActors(); + + if (akkaConfiguration.DistributedPubSubOptions.Enabled) + { + var tracer = provider.GetRequiredService(); + configurationBuilder.StartActors((system, registry) => + { + var cluster = Akka.Cluster.Cluster.Get(system); + cluster.RegisterOnMemberUp(() => + { + system.ActorOf(Props.Create(() => new PingerActor(tracer)), "pinger"); + }); + }); + } + }); + + // enables OpenTracing for ASP.NET Core + services.AddOpenTracing(o => + { + o.ConfigureAspNetCore(a => + { + a.Hosting.OperationNameResolver = context => $"{context.Request.Method} {context.Request.Path}"; + + // skip Prometheus HTTP /metrics collection from appearing in our tracing system + a.Hosting.IgnorePatterns.Add(x => x.Request.Path.StartsWithSegments(new PathString("/metrics"))); + }); + o.ConfigureGenericDiagnostics(c => { }); + }); + + // sets up Prometheus + ASP.NET Core metrics + services.ConfigureAppMetrics(); + + // sets up Jaeger tracing + services.ConfigureJaegerTracing(); + + services.AddRouting(); + services.AddControllers(); + }) + .Configure((context, app) => + { + if (context.HostingEnvironment.IsDevelopment()) + app.UseDeveloperExceptionPage(); + + app.UseRouting(); + + // enable App.Metrics routes + app.UseMetricsAllMiddleware(); + app.UseMetricsAllEndpoints(); + + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + + endpoints.MapGet("/ready", async (ActorRegistry registry) => + { + var readyCheck = registry.Get(); + var checkResult = await readyCheck.Ask(ReadyCheck.Instance, TimeSpan.FromSeconds(3)); + //if (checkResult.IsReady) + return Results.StatusCode(200); + return Results.StatusCode(500); + }); + }); + }) + .Build(); + + await host.RunAsync(); + } + +} \ No newline at end of file diff --git a/src/frontend/OpenTracing.FrontEnd/appsettings.Development.json b/src/frontend/OpenTracing.FrontEnd/appsettings.Development.json new file mode 100644 index 0000000..e203e94 --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Microsoft": "Information" + } + } +} diff --git a/src/frontend/OpenTracing.FrontEnd/appsettings.json b/src/frontend/OpenTracing.FrontEnd/appsettings.json new file mode 100644 index 0000000..f825fc1 --- /dev/null +++ b/src/frontend/OpenTracing.FrontEnd/appsettings.json @@ -0,0 +1,38 @@ +{ + "StressOptions": { + "DistributedPubSubOptions": { + "Enabled": false, + "NumTopics": 100 + }, + "ShardingOptions": { + "Enabled": false, + "UseDData": true, + "RememberEntities": false + }, + "AkkaClusterOptions": { + "Hostname": "localhost", + "Port": 9229, + "ManagementPort": 9228, + "UseKubernetesDiscovery": false, + "UseKubernetesLease": false, + "KubernetesDiscoveryOptions": { + "PodNamespace": "akkastress", + "PodLabelSelector": "akkacluster=stress" + }, + "SeedNodes": ["akka.tcp://LargeNetworkSys@localhost:9221"] + }, + "SerilogOptions": { + "EnableSeq": false + }, + "DispatcherConfig": "Defaults" + }, + + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/src/shared/OpenTracing.Infrastructure/ActorSystemConstants.cs b/src/shared/OpenTracing.Infrastructure/ActorSystemConstants.cs new file mode 100644 index 0000000..5712859 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/ActorSystemConstants.cs @@ -0,0 +1,14 @@ +namespace OpenTracing.Infrastructure; + +public static class ActorSystemConstants +{ + public const string ActorSystemName = "LargeNetworkSys"; + + public const string FrontendRole = "frontend"; + + public const string DistributedPubSubRole = "pubsub"; + + public const string BackendRole = "backend"; + + public const string PingTopicName = "pings"; +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Actors/DispatcherConfigLogger.cs b/src/shared/OpenTracing.Infrastructure/Actors/DispatcherConfigLogger.cs new file mode 100644 index 0000000..b10797a --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Actors/DispatcherConfigLogger.cs @@ -0,0 +1,48 @@ +using System.Text; +using Akka.Actor; +using Akka.Cluster.Sharding; +using Akka.Event; + +namespace OpenTracing.Infrastructure.Actors; + +public sealed class DispatcherConfigLogger : ReceiveActor +{ + private sealed class CheckConfig + { + public static readonly CheckConfig Instance = new CheckConfig(); + private CheckConfig(){} + } + + private readonly ILoggingAdapter _log = Context.GetLogger(); + + public DispatcherConfigLogger() + { + Receive(_ => + { + var sysConfig = Context.System.Settings.Config; + var stringBuilder = new StringBuilder(); + stringBuilder.AppendLine("ALL CONFIGS") + .AppendLine("---- akka.actor.default-dispatcher ----") + .AppendLine(sysConfig.GetConfig("akka.actor.default-dispatcher").ToString(false)) + .AppendLine() + .AppendLine("---- akka.actor.internal-dispatcher ----") + .AppendLine(sysConfig.GetConfig("akka.actor.internal-dispatcher").ToString(false)) + .AppendLine() + .AppendLine("---- akka.remote.default-remote-dispatcher ----") + .AppendLine(sysConfig.GetConfig("akka.remote.default-remote-dispatcher").ToString(false)) + .AppendLine() + .AppendLine("---- akka.remote.backoff-remote-dispatcher ----") + .AppendLine(sysConfig.GetConfig("akka.remote.backoff-remote-dispatcher").ToString(false)); + + _log.Warning(stringBuilder.ToString()); + + var t = typeof(ShardRegion).Assembly; + _log.Warning("Running with version {0} of Akka.Cluster.Sharding", t); + }); + } + + protected override void PreStart() + { + Self.Tell(CheckConfig.Instance); + } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Actors/ItemMessagingActor.cs b/src/shared/OpenTracing.Infrastructure/Actors/ItemMessagingActor.cs new file mode 100644 index 0000000..97ceeb3 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Actors/ItemMessagingActor.cs @@ -0,0 +1,52 @@ +using Akka.Actor; +using Akka.Event; +using Akka.Util; +using AkkaDotNet.Messages; +using AkkaDotNet.Messages.Commands; + +namespace OpenTracing.Infrastructure.Actors; + +public sealed class ItemMessagingActor : ReceiveActor, IWithTimers +{ + private const string ScheduleKey = "writeShard"; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private sealed class WriteShard + { + public static readonly WriteShard Instance = new(); + private WriteShard(){} + } + + public ItemMessagingActor(IActorRef itemShardRegion) + { + _itemShardRegion = itemShardRegion; + + Receive(_ => + { + var productId = ThreadLocalRandom.Current.Next(0, 1_000_000).ToString(); + var shouldWrite = ThreadLocalRandom.Current.Next(0,1); + var countValue = ThreadLocalRandom.Current.Next(0, 10); + + if (shouldWrite == 0) + { + _itemShardRegion.Tell(new AddItem(productId, countValue)); + } + else + { + _itemShardRegion.Tell(new RemoveItem(productId, countValue)); + } + }); + + Receive(resp => + { + _log.Info("Command: {0} resulted in {1}", resp.Command, resp.Result); + }); + } + + protected override void PreStart() + { + Timers!.StartPeriodicTimer(ScheduleKey, WriteShard.Instance, TimeSpan.FromSeconds(1),TimeSpan.FromSeconds(1)); + } + + public ITimerScheduler Timers { get; set; } + private readonly IActorRef _itemShardRegion; +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Actors/ReadyCheckActor.cs b/src/shared/OpenTracing.Infrastructure/Actors/ReadyCheckActor.cs new file mode 100644 index 0000000..30b33a3 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Actors/ReadyCheckActor.cs @@ -0,0 +1,39 @@ +using Akka.Actor; +using Akka.Cluster; + +namespace OpenTracing.Infrastructure.Actors; + +public sealed class ReadyCheck +{ + public static readonly ReadyCheck Instance = new ReadyCheck(); + private ReadyCheck(){} +} + +public sealed class ReadyResult +{ + public ReadyResult(bool isReady) + { + IsReady = isReady; + } + + public bool IsReady { get; } +} + +/// +/// Responsible for processing health-check pings from /ready endpoint. +/// +/// Also used for generating message traffic, optionally, when configured to do so. +/// +public sealed class ReadyCheckActor : ReceiveActor +{ + private readonly Cluster _cluster = Cluster.Get(Context.System); + + public ReadyCheckActor() + { + Receive(r => + { + var isReady = _cluster.SelfMember.Status == MemberStatus.Up && !_cluster.IsTerminated; + Sender.Tell(new ReadyResult(isReady)); + }); + } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Configuration/DispatcherConfig.cs b/src/shared/OpenTracing.Infrastructure/Configuration/DispatcherConfig.cs new file mode 100644 index 0000000..87598ac --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Configuration/DispatcherConfig.cs @@ -0,0 +1,8 @@ +namespace OpenTracing.Infrastructure.Configuration; + +public enum DispatcherConfig +{ + Defaults, + ChannelExecutor64, + DedicatedThreadpool32x16, +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Configuration/StressHostingExtensions.cs b/src/shared/OpenTracing.Infrastructure/Configuration/StressHostingExtensions.cs new file mode 100644 index 0000000..6344735 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Configuration/StressHostingExtensions.cs @@ -0,0 +1,261 @@ +using System.Diagnostics; +using Akka.Actor; +using Akka.Cluster.Hosting; +using Akka.Configuration; +using Akka.Coordination.KubernetesApi; +using Akka.Discovery.KubernetesApi; +using Akka.Hosting; +using Akka.Management.Cluster.Bootstrap; +using Akka.Management.Dsl; +using Akka.Remote.Hosting; +using AkkaDotNet.Messages; +using OpenTracing.Infrastructure.Actors; +using OpenTracing.Infrastructure.Persistence; +using Petabridge.Cmd.Cluster; +using Petabridge.Cmd.Cluster.Sharding; +using Petabridge.Cmd.Host; +using Petabridge.Cmd.Remote; +using Phobos.Hosting; + +namespace OpenTracing.Infrastructure.Configuration; + +/// +/// Designed to add Akka.Cluster.Bootstrap and Akka.Management to our applications +/// +public static class StressHostingExtensions +{ + // class = "Akka.Discovery.KubernetesApi.KubernetesApiServiceDiscovery, Akka.Discovery.KubernetesApi" + public static Config CreateDiscoveryConfig(AkkaClusterOptions options) + { + return $@" + akka{{ + management{{ + http.port = {options.ManagementPort} + http.hostname = """" + cluster.bootstrap {{ + contact-point-discovery {{ + port-name = management + discovery-method = akka.discovery + required-contact-point-nr = 3 + stable-margin = 5s + contact-with-all-contact-points = true + }} + }} + }} + discovery {{ + method = kubernetes-api + + kubernetes-api {{ + class = ""{typeof(KubernetesApiServiceDiscovery).AssemblyQualifiedName}"" + + # Namespace to query for pods. + # Set this value to a specific string to override discovering the namespace using pod-namespace-path. + pod-namespace = ""{options.KubernetesDiscoveryOptions.PodNamespace}"" + + pod-label-selector = ""{options.KubernetesDiscoveryOptions.PodLabelSelector}"" + }} + }} + }} + "; + } + + public static Config DedicatedThreadPoolConfig32x16 = @" + akka.actor.internal-dispatcher { + type = ""Dispatcher"" + executor = ""fork-join-executor"" + throughput = 5 + + fork-join-executor { + parallelism-min = 4 + parallelism-factor = 1.0 + parallelism-max = 32 + } + } + + akka.remote.default-remote-dispatcher { + type = Dispatcher + executor = fork-join-executor + fork-join-executor { + parallelism-min = 2 + parallelism-factor = 0.5 + parallelism-max = 16 + } + } + + akka.remote.backoff-remote-dispatcher { + executor = fork-join-executor + fork-join-executor { + parallelism-min = 2 + parallelism-max = 2 + } + } + "; + + public static Config ChannelExecutorConfig64 = @" + akka.actor.default-dispatcher = { + executor = channel-executor + fork-join-executor { #channelexecutor will re-use these settings + parallelism-min = 2 + parallelism-factor = 1 + parallelism-max = 64 + } + } + + akka.actor.internal-dispatcher = { + executor = channel-executor + throughput = 5 + fork-join-executor { + parallelism-min = 4 + parallelism-factor = 1.0 + parallelism-max = 64 + } + } + + akka.remote.default-remote-dispatcher { + type = Dispatcher + executor = channel-executor + fork-join-executor { + parallelism-min = 2 + parallelism-factor = 0.5 + parallelism-max = 16 + } + } + + akka.remote.backoff-remote-dispatcher { + executor = channel-executor + fork-join-executor { + parallelism-min = 2 + parallelism-max = 2 + } + } + "; + + public static readonly Config MaxFrameSize = @" + akka.remote.dot-netty.tcp.send-buffer-size = 2m + akka.remote.dot-netty.tcp.receive-buffer-size = 2m + akka.remote.dot-netty.tcp.maximum-frame-size = 1m + "; + + /// + /// TODO: can probably incorporate this into Akka.Hosting + /// + public static Config DefaultSbrConfig => @" + akka.cluster{ + downing-provider-class = ""Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster"" + + split-brain-resolver { + active-strategy = keep-majority + down-all-when-unstable = off + } + }"; + + public static Config K8sLeaseSbrConfig => ConfigurationFactory.ParseString(@" + akka.cluster{ + downing-provider-class = ""Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster"" + + split-brain-resolver { + active-strategy = lease-majority + down-all-when-unstable = off + lease-majority { + lease-implementation = ""akka.coordination.lease.kubernetes"" + } + } + } + ").WithFallback(KubernetesLease.DefaultConfiguration); + + public static AkkaConfigurationBuilder WithClusterBootstrap(this AkkaConfigurationBuilder builder, StressOptions options, IEnumerable roles) + { + var clusterOptions = new ClusterOptions() { Roles = roles.ToArray() }; + + if (options.AkkaClusterOptions.UseKubernetesDiscovery) + { + var bootstrapConfig = CreateDiscoveryConfig(options.AkkaClusterOptions) + .WithFallback(ClusterBootstrap.DefaultConfiguration()) + .WithFallback(AkkaManagementProvider.DefaultConfiguration()); + + builder + .AddHocon(bootstrapConfig, HoconAddMode.Prepend) + .WithActors(async (system, registry) => + { + // Akka Management hosts the HTTP routes used by bootstrap + await AkkaManagement.Get(system).Start(); + + // Starting the bootstrap process needs to be done explicitly + await ClusterBootstrap.Get(system).Start(); + }); + } + else + { + // not using K8s discovery - need to populate some seed nodes + if (options.AkkaClusterOptions.SeedNodes != null) + clusterOptions.SeedNodes = options.AkkaClusterOptions.SeedNodes.ToArray(); + } + + switch (options.DispatcherConfig) + { + case DispatcherConfig.ChannelExecutor64: + builder = builder.AddHocon(ChannelExecutorConfig64, HoconAddMode.Prepend); + break; + case DispatcherConfig.DedicatedThreadpool32x16: + builder = builder.AddHocon(DedicatedThreadPoolConfig32x16, HoconAddMode.Prepend); + break; + } + + Debug.Assert(options.AkkaClusterOptions.Port != null, "options.Port != null"); + builder = builder + .AddHocon(options.AkkaClusterOptions.UseKubernetesLease ? K8sLeaseSbrConfig : DefaultSbrConfig, HoconAddMode.Prepend) // need to add SBR + .AddHocon(MaxFrameSize, HoconAddMode.Prepend) + .WithRemoting(options.AkkaClusterOptions.Hostname, options.AkkaClusterOptions.Port.Value) + .WithClustering(clusterOptions) + .AddPersistence(options.PersistenceOptions) + .WithPhobos(AkkaRunMode.AkkaCluster, configBuilder => + { + configBuilder.WithTracing(tracingConfigBuilder => + { + tracingConfigBuilder.SetTraceUserActors(false).SetTraceSystemActors(false); + }); + }) + .StartActors((system, registry) => + { + system.ActorOf(Props.Create(() => new DispatcherConfigLogger())); + }) + .WithPetabridgeCmd(); // start PetabridgeCmd actors too + + return builder; + } + + public static AkkaConfigurationBuilder WithReadyCheckActors(this AkkaConfigurationBuilder builder) + { + return builder.StartActors((system, registry) => + { + var readyCheckActor = system.ActorOf(Props.Create(() => new ReadyCheckActor()), "ready"); + registry.TryRegister(readyCheckActor); + }); + } + + public static AkkaConfigurationBuilder WithItemMessagingActor(this AkkaConfigurationBuilder builder) + { + return builder.StartActors((system, registry) => + { + var cluster = Akka.Cluster.Cluster.Get(system); + + cluster.RegisterOnMemberUp(() => + { + var shardRegion = registry.Get(); + system.ActorOf(Props.Create(() => new ItemMessagingActor(shardRegion)), "item-messaging"); + }); + }); + } + + public static AkkaConfigurationBuilder WithPetabridgeCmd(this AkkaConfigurationBuilder builder) + { + return builder.StartActors((system, registry) => + { + var petabridgeCmd = PetabridgeCmd.Get(system); + petabridgeCmd.RegisterCommandPalette(ClusterCommands.Instance); + petabridgeCmd.RegisterCommandPalette(ClusterShardingCommands.Instance); + petabridgeCmd.RegisterCommandPalette(new RemoteCommands()); + petabridgeCmd.Start(); + }); + } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Configuration/StressOptions.cs b/src/shared/OpenTracing.Infrastructure/Configuration/StressOptions.cs new file mode 100644 index 0000000..94764a6 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Configuration/StressOptions.cs @@ -0,0 +1,61 @@ +using OpenTracing.Infrastructure.Logging; +using OpenTracing.Infrastructure.Persistence; + +namespace OpenTracing.Infrastructure.Configuration; + +public class StressOptions +{ + public bool EnablePhobos { get; set; } = false; + + public DistributedPubSubOptions DistributedPubSubOptions { get; set; } = new DistributedPubSubOptions(); + public ShardingOptions ShardingOptions { get; set; } = new ShardingOptions(); + public AkkaClusterOptions AkkaClusterOptions { get; set; } = new AkkaClusterOptions(); + + public SerilogOptions SerilogOptions { get; set; } = new SerilogOptions(); + + public DispatcherConfig DispatcherConfig { get; set; } = DispatcherConfig.Defaults; + + public PersistenceOptions PersistenceOptions { get; set; } = new PersistenceOptions(); +} + +public class DistributedPubSubOptions +{ + public bool Enabled { get; set; } + public int NumTopics { get; set; } +} + +public class ShardingOptions +{ + public bool Enabled { get; set; } + public bool UseDData { get; set; } + public bool RememberEntities { get; set; } +} + +public class AkkaClusterOptions +{ + public string? Hostname { get; set; } + public int? Port { get;set; } + + /// + /// Port used by Akka.Management HTTP + /// + public int? ManagementPort { get; set; } + + public List? Roles { get; set; } + public bool UseKubernetesDiscovery { get; set; } = false; + + public bool UseKubernetesLease { get; set; } = false; + + public KubernetesDiscoveryOptions KubernetesDiscoveryOptions { get; set; } = new KubernetesDiscoveryOptions(); + + /// + /// Used when we aren't doing Kubernetes discovery + /// + public List? SeedNodes { get; set; } +} + +public class KubernetesDiscoveryOptions +{ + public string PodNamespace { get; set; } = string.Empty; + public string PodLabelSelector { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Extensions.cs b/src/shared/OpenTracing.Infrastructure/Extensions.cs new file mode 100644 index 0000000..9220c9e --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Extensions.cs @@ -0,0 +1,107 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2015-2023 .NET Petabridge, LLC +// +// ----------------------------------------------------------------------- + +using System.Net; +using System.Reflection; +using App.Metrics; +using App.Metrics.Formatters.Prometheus; +using Jaeger; +using Jaeger.Reporters; +using Jaeger.Samplers; +using Jaeger.Senders; +using Jaeger.Senders.Thrift; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Phobos.Tracing.Scopes; + +namespace OpenTracing.Infrastructure; + +public static class Extensions +{ + /// + /// Name of the variable used to direct Phobos' Jaeger + /// output. + /// See https://github.com/jaegertracing/jaeger-client-csharp for details. + /// + public const string JaegerAgentHostEnvironmentVar = "JAEGER_AGENT_HOST"; + + public const string JaegerEndpointEnvironmentVar = "JAEGER_ENDPOINT"; + + public const string JaegerAgentPortEnvironmentVar = "JAEGER_AGENT_PORT"; + + public const int DefaultJaegerAgentPort = 6832; + + public static void ConfigureAppMetrics(this IServiceCollection services) + { + services.AddMetricsTrackingMiddleware(); + services.AddMetrics(b => + { + var metrics = b.Configuration.Configure(o => + { + o.GlobalTags.Add("host", Dns.GetHostName()); + o.DefaultContextLabel = "akka.net"; + o.Enabled = true; + o.ReportingEnabled = true; + }) + .OutputMetrics.AsPrometheusPlainText() + .Build(); + + services.AddMetricsEndpoints(ep => + { + ep.MetricsTextEndpointOutputFormatter = metrics.OutputMetricsFormatters + .OfType().First(); + ep.MetricsEndpointOutputFormatter = metrics.OutputMetricsFormatters + .OfType().First(); + }); + }); + services.AddMetricsReportingHostedService(); + } + + public static void ConfigureJaegerTracing(this IServiceCollection services) + { + static ISender BuildSender() + { + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(JaegerEndpointEnvironmentVar))) + { + if (!int.TryParse(Environment.GetEnvironmentVariable(JaegerAgentPortEnvironmentVar), + out var udpPort)) + udpPort = DefaultJaegerAgentPort; + return new UdpSender( + Environment.GetEnvironmentVariable(JaegerAgentHostEnvironmentVar) ?? "localhost", + udpPort, 0); + } + + return new HttpSender(Environment.GetEnvironmentVariable(JaegerEndpointEnvironmentVar)); + } + + services.AddSingleton(sp => + { + var loggerFactory = sp.GetRequiredService(); + + var builder = BuildSender(); + var logReporter = new LoggingReporter(loggerFactory); + + var remoteReporter = new RemoteReporter.Builder() + .WithLoggerFactory(loggerFactory) // optional, defaults to no logging + .WithMaxQueueSize(100) // optional, defaults to 100 + .WithFlushInterval(TimeSpan.FromSeconds(1)) // optional, defaults to TimeSpan.FromSeconds(1) + .WithSender(builder) // optional, defaults to UdpSender("localhost", 6831, 0) + .Build(); + + var sampler = new ConstSampler(true); // keep sampling disabled + + // name the service after the executing assembly + var tracer = new Tracer.Builder(Assembly.GetExecutingAssembly().GetName().Name) + .WithReporter(new CompositeReporter(remoteReporter, logReporter)) + .WithSampler(sampler) + // IMPORTANT: ActorScopeManager needed to properly correlate trace inside Akka.NET + .WithScopeManager(new ActorScopeManager()); + + return tracer.Build(); + }); + } + +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Logging/SerilogConfigurationExtensions.cs b/src/shared/OpenTracing.Infrastructure/Logging/SerilogConfigurationExtensions.cs new file mode 100644 index 0000000..6ab4fb7 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Logging/SerilogConfigurationExtensions.cs @@ -0,0 +1,73 @@ +using System.Net; +using System.Reflection; +using Akka.Configuration; +using Akka.Hosting; +using Serilog; +using Serilog.Events; +using Serilog.Sinks.SystemConsole.Themes; + +namespace OpenTracing.Infrastructure.Logging; + +public static class SerilogConfigurationExtensions +{ + public const string ServiceNameProperty = "SERVICE_NAME"; + public const string PodNameProperty = "POD_NAME"; + + public static readonly Config SerilogConfig = + @" + akka.loglevel = INFO + akka.loggers =[""Akka.Logger.Serilog.SerilogLogger, Akka.Logger.Serilog""]"; + + public static string GetServiceName() + { + var podName = Environment.GetEnvironmentVariable(PodNameProperty); + + return !string.IsNullOrEmpty(podName) ? podName : Dns.GetHostName(); + } + + + public static AkkaConfigurationBuilder WithSerilog(this AkkaConfigurationBuilder builder, SerilogOptions options) + { + var loggerConfiguration = new LoggerConfiguration() + .Enrich.FromLogContext() + .Enrich.WithProperty(PodNameProperty, GetServiceName()) + .Enrich.WithProperty(ServiceNameProperty, Assembly.GetEntryAssembly()?.GetName().Name) + .WriteTo.Console( + outputTemplate: + "[{SERVICE_NAME}][{POD_NAME}][{Timestamp:HH:mm:ss} {Level}] {SourceContext}{NewLine}{Message:lj}{NewLine}{Exception}{NewLine}", + theme: AnsiConsoleTheme.Literate) + .Filter.ByExcluding(ExcludeHealthChecksNormalEvents).MinimumLevel.Debug(); // Do not want lots of health check info logs in console + + if (options.EnableSeq) + { + var serverUrl = $"http://{options.SeqHost}:{options.SeqPort}"; + Console.WriteLine($"Writing logs to {serverUrl}"); + loggerConfiguration = loggerConfiguration + .WriteTo.Seq(serverUrl) + .Filter.ByExcluding(ExcludeHealthChecksNormalEvents); // Do not want lots of health check info logs in Seq + } + + // Configure Serilog + Log.Logger = loggerConfiguration.CreateLogger(); + + // add to Akka.NET + return builder.AddHocon(SerilogConfig, HoconAddMode.Prepend); + } + + /// + /// Filter out health check noise from Seq + /// + /// The log event to filter + /// true if we're going to exclude this event. + private static bool ExcludeHealthChecksNormalEvents(LogEvent ev) + { + var healthCheckRequest = ev.Properties.ContainsKey("RequestPath") && + (ev.Properties["RequestPath"].ToString() == "\"/env\"" || + ev.Properties["RequestPath"].ToString() == "\"/ready\""); + + var metricsLog = ev.Properties.ContainsKey("kubernetes_annotations_prometheus.io_path") && + ev.Properties["kubernetes_annotations_prometheus.io_path"].ToString() == "\"/metrics\""; + + return ev.Level < LogEventLevel.Warning && (healthCheckRequest || metricsLog); + } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Logging/SerilogOptions.cs b/src/shared/OpenTracing.Infrastructure/Logging/SerilogOptions.cs new file mode 100644 index 0000000..e51a064 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Logging/SerilogOptions.cs @@ -0,0 +1,8 @@ +namespace OpenTracing.Infrastructure.Logging; + +public class SerilogOptions +{ + public bool EnableSeq { get; set; } = false; + public string? SeqHost { get; set; } + public int? SeqPort { get; set; } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/OpenTracing.Infrastructure.csproj b/src/shared/OpenTracing.Infrastructure/OpenTracing.Infrastructure.csproj new file mode 100644 index 0000000..66b9d1f --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/OpenTracing.Infrastructure.csproj @@ -0,0 +1,44 @@ + + + + $(NetCoreFramework) + enable + enable + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceHostingExtensions.cs b/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceHostingExtensions.cs new file mode 100644 index 0000000..e0e9323 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceHostingExtensions.cs @@ -0,0 +1,42 @@ +using Akka.Configuration; +using Akka.Hosting; +using Akka.Persistence.Azure; + +namespace OpenTracing.Infrastructure.Persistence; + +public static class PersistenceHostingExtensions +{ + public static Config GetPersistenceHocon(string connectionString) + { + return $@" + akka.persistence {{ + journal {{ + plugin = ""akka.persistence.journal.azure-table"" + azure-table {{ + class = ""Akka.Persistence.Azure.Journal.AzureTableStorageJournal, Akka.Persistence.Azure"" + connection-string = ""{connectionString}"" + }} + }} + snapshot-store {{ + plugin = ""akka.persistence.snapshot-store.azure-blob-store"" + azure-blob-store {{ + class = ""Akka.Persistence.Azure.Snapshot.AzureBlobSnapshotStore, Akka.Persistence.Azure"" + connection-string = ""{connectionString}"" + }} + }} + }}"; + } + + public static AkkaConfigurationBuilder AddPersistence(this AkkaConfigurationBuilder configurationBuilder, + PersistenceOptions options) + { + if (options.Enabled) + { + var persistenceHocon = GetPersistenceHocon(options.AzureStorageConnectionString) + .WithFallback(AzurePersistence.DefaultConfig); + configurationBuilder.AddHocon(persistenceHocon, HoconAddMode.Prepend); + } + + return configurationBuilder; + } +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceOptions.cs b/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceOptions.cs new file mode 100644 index 0000000..56728c7 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Persistence/PersistenceOptions.cs @@ -0,0 +1,8 @@ +namespace OpenTracing.Infrastructure.Persistence; + +public class PersistenceOptions +{ + public bool Enabled { get; set; } = false; + + public string AzureStorageConnectionString { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/src/shared/OpenTracing.Infrastructure/Sharding/ItemShardExtractor.cs b/src/shared/OpenTracing.Infrastructure/Sharding/ItemShardExtractor.cs new file mode 100644 index 0000000..5178d19 --- /dev/null +++ b/src/shared/OpenTracing.Infrastructure/Sharding/ItemShardExtractor.cs @@ -0,0 +1,22 @@ +using Akka.Cluster.Sharding; +using AkkaDotNet.Messages; + +namespace OpenTracing.Infrastructure.Sharding; + +public class ItemShardExtractor : HashCodeMessageExtractor +{ + // 200 nodes, 10 shards per node + public ItemShardExtractor() : base(2000) + { + } + + public override string? EntityId(object message) + { + if (message is IWithItem itemId) + { + return itemId.ItemId; + } + + return null; + } +} \ No newline at end of file