From 8b86e5385204ed429021d2d3e78752ca1b1602fd Mon Sep 17 00:00:00 2001 From: DoTheEvo Date: Sun, 19 Mar 2023 14:09:36 +0100 Subject: [PATCH] update --- .../dashboards/docker_containers.json | 1270 ++++++ .../dashboards/docker_host.json | 1441 +++++++ .../dashboards/monitoring_services.json | 3412 +++++++++++++++++ prometheus_grafana_loki/readme.md | 1582 ++++++++ 4 files changed, 7705 insertions(+) create mode 100644 prometheus_grafana_loki/dashboards/docker_containers.json create mode 100644 prometheus_grafana_loki/dashboards/docker_host.json create mode 100644 prometheus_grafana_loki/dashboards/monitoring_services.json create mode 100644 prometheus_grafana_loki/readme.md diff --git a/prometheus_grafana_loki/dashboards/docker_containers.json b/prometheus_grafana_loki/dashboards/docker_containers.json new file mode 100644 index 0000000..9589e7c --- /dev/null +++ b/prometheus_grafana_loki/dashboards/docker_containers.json @@ -0,0 +1,1270 @@ +{ + "id": null, + "title": "Docker Containers", + "description": "Containers metrics", + "tags": [ + "docker" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": true, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "150px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\"}[1m])) / count(node_cpu_seconds_total{mode=\"user\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Load", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "machine_cpu_cores", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "machine_cpu_cores", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "(sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "65, 90", + "title": "Memory Load", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\"})", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "timeFrom": "10s", + "title": "Used Memory", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "(node_filesystem_size_bytes{fstype=\"ext4\"} - node_filesystem_free_bytes{fstype=\"ext4\"}) / node_filesystem_size_bytes{fstype=\"ext4\"} * 100", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": "65, 90", + "title": "Storage Load", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 3, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(container_fs_usage_bytes)", + "interval": "30s", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Used Storage", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + } + ], + "title": "Overview" + }, + { + "collapse": false, + "editable": true, + "height": "150px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "datasource": "Prometheus", + "decimals": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "id": 9, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(count(container_memory_usage_bytes{image!=\"\"}) > 0)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "containers", + "refId": "A", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Running Containers", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "load 1m", + "color": "#BF1B00" + } + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load 1m", + "metric": "node_load1", + "refId": "A", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System Load", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 15, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "written", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_disk_read_bytes_total[5m]))", + "interval": "2s", + "intervalFactor": 4, + "legendFormat": "read", + "metric": "", + "refId": "A", + "step": 8 + }, + { + "expr": "sum(irate(node_disk_written_bytes_total[5m]))", + "interval": "2s", + "intervalFactor": 4, + "legendFormat": "written", + "metric": "", + "refId": "B", + "step": 8 + }, + { + "expr": "sum(irate(node_disk_io_time_seconds_total[5m]))", + "interval": "2s", + "intervalFactor": 4, + "legendFormat": "io time", + "metric": "", + "refId": "C", + "step": 8 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "I/O Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Host stats" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 8, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m])) / scalar(count(node_cpu_seconds_total{mode=\"user\"})) * 100", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_cpu_user_seconds_total", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "CPU" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 11, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name)(container_memory_usage_bytes{image!=\"\",container_label_org_label_schema_group=\"\"})", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "metric": "container_memory_usage", + "refId": "A", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Container Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 12, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (container_memory_cache{image!=\"\",container_label_org_label_schema_group=\"\"})", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_cache", + "refId": "A", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Container Cached Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Memory" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 13, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_network_receive_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Container Network Input", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 14, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_network_transmit_bytes_total", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Container Network Output", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Network" + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": "10s", + "schemaVersion": 12, + "version": 8, + "links": [], + "gnetId": null +} diff --git a/prometheus_grafana_loki/dashboards/docker_host.json b/prometheus_grafana_loki/dashboards/docker_host.json new file mode 100644 index 0000000..a6beef7 --- /dev/null +++ b/prometheus_grafana_loki/dashboards/docker_host.json @@ -0,0 +1,1441 @@ +{ + "id": null, + "title": "Docker Host", + "description": "Docker host metrics", + "tags": [ + "system" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": true, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "100px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 1, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 1, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "s", + "postfixFontSize": "80%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "node_time_seconds - node_boot_time_seconds", + "interval": "30s", + "intervalFactor": 1, + "refId": "A", + "step": 30 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "CPU Idle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "machine_cpu_cores", + "intervalFactor": 2, + "metric": "machine_cpu_cores", + "refId": "A", + "step": 2 + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes", + "interval": "30s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Available Memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 3, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "node_memory_SwapFree_bytes", + "interval": "30s", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Free Swap", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(node_filesystem_free_bytes{fstype=\"ext4\"})", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": "", + "title": "Free Storage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg", + "timeFrom": "10s", + "hideTimeOverride": true + } + ], + "title": "Available resources" + }, + { + "collapse": false, + "editable": true, + "height": "150px", + "panels": [ + { + "aliasColors": {}, + "bars": true, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 9, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "load 1m", + "color": "#1F78C1" + } + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "load 1m", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average 1m", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "blocked by I/O", + "color": "#58140C" + } + ], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_procs_running", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "running", + "metric": "node_procs_running", + "refId": "A", + "step": 10 + }, + { + "expr": "node_procs_blocked", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "blocked by I/O", + "metric": "node_procs_blocked", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Processes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 11, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "interrupts", + "color": "#806EB7" + } + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " irate(node_intr_total[5m])", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "interrupts", + "metric": "node_intr_total", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Interrupts", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Load" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_cpu_seconds_total[1m])) by (mode) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))", + "intervalFactor": 10, + "legendFormat": "{{ mode }}", + "metric": "node_cpu_seconds_total", + "refId": "A", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "title": "CPU" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 6, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Used", + "color": "#BF1B00" + }, + { + "alias": "Free", + "color": "#7EB26D" + }, + { + "alias": "Buffers", + "color": "#6ED0E0" + }, + { + "alias": "Cached", + "color": "#EF843C" + } + ], + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 1 + }, + { + "expr": "node_memory_MemFree_bytes", + "intervalFactor": 1, + "legendFormat": "Free", + "refId": "B", + "step": 1 + }, + { + "expr": "node_memory_Buffers_bytes", + "intervalFactor": 1, + "legendFormat": "Buffers", + "refId": "C", + "step": 1 + }, + { + "expr": "node_memory_Cached_bytes", + "intervalFactor": 1, + "legendFormat": "Cached", + "refId": "D", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Memory" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 7, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "written", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_disk_read_bytes_total[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "read", + "metric": "node_disk_read_bytes_total", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(irate(node_disk_written_bytes_total[1m]))", + "intervalFactor": 1, + "legendFormat": "written", + "metric": "node_disk_written_bytes_total", + "refId": "B", + "step": 1 + }, + { + "expr": "sum(irate(node_disk_io_time_seconds_total[1m]))", + "intervalFactor": 1, + "legendFormat": "io time", + "metric": "node_disk_io_time_seconds_total", + "refId": "C", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "I/O Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "I/O" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 8, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "In: {{ device }}", + "metric": "node_network_receive_bytes_total", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(container_network_transmit_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))", + "intervalFactor": 1, + "legendFormat": "Out: {{ device }}", + "metric": "node_network_transmit_bytes_total", + "refId": "B", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Network" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 14, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Used", + "color": "#890F02" + }, + { + "alias": "Free", + "color": "#7EB26D" + } + ], + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 10 + }, + { + "expr": "node_memory_SwapFree_bytes", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Free", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Swap Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 15, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_vmstat_pswpin[1m]) * 4096 or irate(node_vmstat_pswpin[5m]) * 4096", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "In", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(node_vmstat_pswpout[1m]) * 4096 or irate(node_vmstat_pswpout[5m]) * 4096", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Out", + "refId": "B", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Swap I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": "10s", + "schemaVersion": 12, + "version": 2, + "links": [], + "gnetId": null +} diff --git a/prometheus_grafana_loki/dashboards/monitoring_services.json b/prometheus_grafana_loki/dashboards/monitoring_services.json new file mode 100644 index 0000000..00df64a --- /dev/null +++ b/prometheus_grafana_loki/dashboards/monitoring_services.json @@ -0,0 +1,3412 @@ +{ + "id": null, + "title": "Monitoring Services", + "tags": [ + "prometheus" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": true, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 1, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "s", + "postfixFontSize": "80%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(time() - process_start_time_seconds{instance=\"localhost:9090\",job=\"prometheus\"})", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "timeFrom": "10s", + "timeShift": null, + "title": "Prometheus Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "timeFrom": "10s", + "timeShift": null, + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "hideTimeOverride": true, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "timeFrom": "10s", + "timeShift": null, + "title": "In-Memory Chunks", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "hideTimeOverride": true, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_tsdb_head_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "timeFrom": "10s", + "timeShift": null, + "title": "In-Memory Series", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu{mode=\"user\"})))", + "format": "time_series", + "hide": true, + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"})))", + "format": "time_series", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 73, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ALERTS{alertstate=\"firing\"}) by (alertname)", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{ alertname }}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "repeat": null, + "title": "Prometheus Metrics", + "type": "row" + }, + { + "aliasColors": { + "Max": "#e24d42", + "Open": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "A" + }, + { + "expr": "process_open_fds{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#7EB26D", + "Allocated bytes - 1m max": "#BF1B00", + "Allocated bytes - 1m min": "#BF1B00", + "Allocated bytes - 5m max": "#BF1B00", + "Allocated bytes - 5m min": "#BF1B00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#447EBC" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/-/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RSS", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 10 + }, + { + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Target heap size", + "metric": "go_memstats_alloc_bytes", + "refId": "D", + "step": 10 + }, + { + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Next GC", + "metric": "go_memstats_next_gc_bytes", + "refId": "C", + "step": 10 + }, + { + "expr": "go_memstats_alloc_bytes{job=\"prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Allocations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Time series": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time series", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Time series", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 32 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_active_appenders", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head Appenders", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Active Appenders", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "samples/s": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 32 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Samples", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Samples Appended", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "To persist": "#9AC48A" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunks", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_created_total[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Created", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks Created", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Removed": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_removed_total[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Removed", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks Removed", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max": "#447ebc", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Min": "#447ebc", + "Now": "#7eb26d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_min_time", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Min", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "time() * 1000", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Now", + "refId": "C" + }, + { + "expr": "prometheus_tsdb_head_max_time", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GC Time", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head GC Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Queue length", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_blocks_loaded", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks Loaded", + "metric": "prometheus_local_storage_indexing_batch_sizes_sum", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Blocks Loaded", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Failed Reloads": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_reloads_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reloads", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Reloads", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_corruptions_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Corruptions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_reloads_failures_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reload Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_series_not_found[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Head Series Not Found", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "C", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_compactions_failed_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compaction Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "D", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Retention Cutoff Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Problems", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Fsync Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Truncate Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "WAL Latencies", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compactions_total[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compactions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Compactions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 60 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compaction Time", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Compaction Time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 60 + }, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_total[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Retention Cutoffs", + "metric": "last", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Retention Cutoffs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 67 + }, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Samples", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "First Compaction, Avg Chunk Samples", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 67 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_count[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ interval }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Target Scrapes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 67 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} ({{interval}})", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 74 + }, + "id": 62, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP requests", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 74 + }, + "id": 64, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum[1m]) / rate(prometheus_http_request_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP request latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 74 + }, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time spent in HTTP requests", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Time spent in each mode, per second", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 81 + }, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query engine timings", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 81 + }, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_iterations_missed_total[1m]) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rule group missed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_rule_evaluation_failures_total[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rule evals failed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rule group evaulation problems", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 81 + }, + "id": 72, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_duration_seconds_sum[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rule evaluation duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Evaluation time of rule groups", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": "10s", + "schemaVersion": 12, + "version": 22, + "links": [], + "gnetId": null +} diff --git a/prometheus_grafana_loki/readme.md b/prometheus_grafana_loki/readme.md new file mode 100644 index 0000000..73e256d --- /dev/null +++ b/prometheus_grafana_loki/readme.md @@ -0,0 +1,1582 @@ +# Prometheus+Grafana in docker + +###### guide-by-example + +![logo](https://i.imgur.com/q41QfyI.png) + +# Purpose + +Monitoring of a docker host and the cointaners. + +* [Official Prometheus](https://prometheus.io/) +* [Official Grafana](https://grafana.com/) +* [Official Loki](https://grafana.com/oss/loki/) + +Monitoring in this case means gathering and showing information on how services +or machines or containers are running. Can be cpu, io, ram, disk use... +can be number of http requests, errors, or results of backups.
+Prometheus deals with metrics. Loki deals with logs. Grafana is there to show +the data on a dashboard. + +Lot of the prometheus stuff here is based off the magnificent +[stefanprodan/dockprom.](https://github.com/stefanprodan/dockprom) + +# Chapters + +* **[Core prometheus+grafana](#Overview)** - nice dashboards with metrics of docker host and containers +* **[PromQL](#PromQL)** - links to various learning resources +* **[Pushgateway](#Pushgateway)** - push data to prometheus from anywhere +* **[Alertmanager](#Alertmanager)** - setting alerts and getting notifications +* **[Loki](#Loki)** - prometheus for logs +* **[Minecraft Loki example](#minecraft-loki-example)** - logs, grafana alerts + and templates +* **[Caddy reverse proxy monitoring](#caddy-reverse-proxy-monitoring)** - + metrics, logs and geoip map + +![dashboards_pic](https://i.imgur.com/ZmyP0T8.png) + +# Overview + +[Good youtube overview](https://youtu.be/h4Sl21AKiDg) of Prometheus.
+ +Prometheus is an open source system for monitoring and alerting, +written in golang.
+It periodically collects **metrics** from configured **targets**, +makes these metrics available for visualization, and can trigger **alerts**.
+Prometheus is relatively young project, it is a **pull type** monitoring. + +[Glossary.](https://prometheus.io/docs/introduction/glossary/) + +* **Prometheus Server** is the core of the system, responsible for + * pulling new metrics + * storing the metrics in a database and evaluating them + * making metrics available through PromQL API +* **Targets** - machines, services, applications that are monitored.
+ These need to have an **exporter**. + * **exporter** - a script or a service that gathers metrics on the target, + converts them to prometheus server format, + and exposes them at an endpoint so they can be pulled +* **Alertmanager** - responsible for handling alerts from Prometheus Server, + and sending notifications through email, slack, pushover,.. + **In this setup [ntfy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal) + webhook will be used.** +* **pushgateway** - allows push type of monitoring. Meaning a machine anywhere + in the world can push data in to your prometheus. Should not be overused + as it goes against the pull philosophy of prometheus. +* **Grafana** - for web UI visualization of the collected metrics + + +![prometheus components](https://i.imgur.com/AxJCg8C.png) + +# Files and directory structure + +``` +/home/ + └── ~/ + └── docker/ + └── prometheus/ + ├── 🗁 grafana_data/ + ├── 🗁 prometheus_data/ + ├── 🗋 docker-compose.yml + ├── 🗋 .env + └── 🗋 prometheus.yml +``` + +* `grafana_data/` - a directory where grafana stores its data +* `prometheus_data/` - a directory where prometheus stores its database and data +* `.env` - a file containing environment variables for docker compose +* `docker-compose.yml` - a docker compose file, telling docker how to run the containers +* `prometheus.yml` - a configuration file for prometheus + +The three files must be provided.
+The directories are created by docker compose on the first run. + +# docker-compose + +* **Prometheus** - The official image used. Few extra commands passing configuration. + Of note is 240 hours(10days) **retention** policy. +* **Grafana** - The official image used. Bind mounted directory + for persistent data storage. User sets **as root**, as it solves issues I am + lazy to investigate. +* **NodeExporter** - An exporter for linux machines, + in this case gathering the **metrics** of the docker host, + like uptime, cpu load, memory use, network bandwidth use, disk space,...
+ Also **bind mount** of some system directories to have access to required info. +* **cAdvisor** - An exporter for gathering docker **containers** metrics, + showing cpu, memory, network use of **each container**
+ Runs in `privileged` mode and has some bind mounts of system directories + to have access to required info. + +*Note* - ports are only `expose`, since expectation of use of a reverse proxy +and accessing the services by hostname, not ip and port. + +`docker-compose.yml` +```yml +services: + + # MONITORING SYSTEM AND THE METRICS DATABASE + prometheus: + image: prom/prometheus:v2.42.0 + container_name: prometheus + hostname: prometheus + user: root + restart: unless-stopped + depends_on: + - cadvisor + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=240h' + - '--web.enable-lifecycle' + volumes: + - ./prometheus_data:/prometheus + - ./prometheus.yml:/etc/prometheus/prometheus.yml + expose: + - "9090" + labels: + org.label-schema.group: "monitoring" + + # WEB BASED UI VISUALISATION OF METRICS + grafana: + image: grafana/grafana:9.4.3 + container_name: grafana + hostname: grafana + user: root + restart: unless-stopped + env_file: .env + volumes: + - ./grafana_data:/var/lib/grafana + expose: + - "3000" + labels: + org.label-schema.group: "monitoring" + + # HOST LINUX MACHINE METRICS EXPORTER + nodeexporter: + image: prom/node-exporter:v1.5.0 + container_name: nodeexporter + hostname: nodeexporter + restart: unless-stopped + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + expose: + - "9100" + labels: + org.label-schema.group: "monitoring" + + # DOCKER CONTAINERS METRICS EXPORTER + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.47.1 + container_name: cadvisor + hostname: cadvisor + restart: unless-stopped + privileged: true + devices: + - /dev/kmsg:/dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux + expose: + - "3000" + labels: + org.label-schema.group: "monitoring" + +networks: + default: + name: $DOCKER_MY_NETWORK + external: true +``` + +`.env` + +```bash +# GENERAL +DOCKER_MY_NETWORK=caddy_net +TZ=Europe/Bratislava + +# GRAFANA +GF_SECURITY_ADMIN_USER=admin +GF_SECURITY_ADMIN_PASSWORD=admin +GF_USERS_ALLOW_SIGN_UP=false +# GRAFANA EMAIL +GF_SMTP_ENABLED=true +GF_SMTP_HOST=smtp-relay.sendinblue.com:587 +GF_SMTP_USER=example@gmail.com +GF_SMTP_PASSWORD=xzu0dfFhn3eqa +``` + +**All containers must be on the same network**.
+Which is named in the `.env` file.
+If one does not exist yet: `docker network create caddy_net` + +## prometheus.yml + +[Official documentation.](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) + +Contains the bare minimum setup of targets from where metrics are to be pulled. + +`prometheus.yml` +```yml +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'nodeexporter' + static_configs: + - targets: ['nodeexporter:9100'] + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] +``` + +## Reverse proxy + +Caddy v2 is used, details +[here](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy_v2).
+ +`Caddyfile` +```php +graf.{$MY_DOMAIN} { + reverse_proxy grafana:3000 +} + +prom.{$MY_DOMAIN} { + reverse_proxy prometheus:9090 +} +``` + +## First run and Grafana configuration + +* login admin/admin to `graf.example.com`, change the password +* add Prometheus as a Data source in configuration
+ set URL to `http://prometheus:9090`
+* import dashboards from [json files in this repo](dashboards/)
+ +These **dashboards** are the preconfigured ones from +[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom) +with **few changes**.
+**Docker host** dashboard did not show free disk space for me, **had to change fstype** +from `aufs` to `ext4`. +Also included is [a fix](https://github.com/stefanprodan/dockprom/issues/18#issuecomment-487023049) +for **host network monitoring** not showing traffick. In all of them +the default time interval is set to 1h instead of 15m + +* **docker_host.json** - dashboard showing linux docker host metrics +* **docker_containers.json** - dashboard showing docker containers metrics, + except the ones labeled as `monitoring` in the compose file +* **monitoring_services.json** - dashboar showing docker containers metrics + of containers that are labeled `monitoring` + +![interface-pic](https://i.imgur.com/wzwgBkp.png) + +--- +--- + +# PromQL + +Some concept, highlights and examples. + +* [Stackoverflow - Prometheus instant vector vs range vector](https://stackoverflow.com/questions/68223824/prometheus-instant-vector-vs-range-vector) +* [Short video](https://youtu.be/yLPTHinHB6Y) +* [Prometheus Cheat Sheet - Basics \(Metrics, Labels, Time Series, Scraping\)](https://iximiuz.com/en/posts/prometheus-metrics-labels-time-series/) +* [Learning Prometheus and PromQL - Learning Series](https://iximiuz.com/en/series/learning-prometheus-and-promql/) +* [The official](https://prometheus.io/docs/prometheus/latest/querying/basics/) + +One thing to get from these is what kind of data a query in PromQL returns. +Instant verctor vs range vector. + +--- +--- + +# Pushgateway + +Gives freedom to **push** information in to prometheus from **anywhere**. + +### The setup + +To **add** pushgateway functionality to the current stack: + +* **New container** `pushgateway` added to the **compose** file. + +
+ docker-compose.yml + + ```yml + services: + + # PUSHGATEWAY FOR PROMETHEUS + pushgateway: + image: prom/pushgateway:v1.5.1 + container_name: pushgateway + hostname: pushgateway + restart: unless-stopped + command: + - '--web.enable-admin-api' + expose: + - "9091" + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +* Adding pushgateway to the **Caddyfile** of the reverse proxy so that + it can be reached at `https://push.example.com`
+ +
+ Caddyfile + + ```php + push.{$MY_DOMAIN} { + reverse_proxy pushgateway:9091 + } + ``` +
+ +* Adding pushgateway's **scrape point** to `prometheus.yml`
+ +
+ prometheus.yml + + ```yml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'pushgateway-scrape' + honor_labels: true + static_configs: + - targets: ['pushgateway:9091'] + ``` +
+ +### The basics + +![veeam-dash](https://i.imgur.com/TOuv9bM.png) + +To **test pushing** some metric, execute in linux:
+`echo "some_metric 3.14" | curl --data-binary @- https://push.example.com/metrics/job/blabla/instance/whatever` + +You see **labels** being set to the pushed metric in the path.
+Label `job` is required, but after that it's whatever you want, +though use of `instance` label is customary.
+Now in grafana, in **Explore** section you should see some results +when quering for `some_metric`. + +The metrics sit on the pushgateway **forever**, unless deleted or container +shuts down. **Prometheus will not remove** the metrics from it **after scraping**, +it will keep scraping the pushgateway and store the value that sits there with +the time of scraping. + +To **wipe** the pushgateway clean
+`curl -X PUT https://push.example.com/api/v1/admin/wipe` + +### The real world use + +[**Veeam Prometheus Grafana - guide-by-example**](https://github.com/DoTheEvo/veeam-prometheus-grafana) + +Linked above is much more on **pushgateway setup**, +a real world use to **monitor backups**, along with **pushing metrics +from windows** in powershell.
+ +![veeam-dash](https://i.imgur.com/dUyzuyl.png) + +--- +--- + +# Alertmanager + +To send a **notification** about some **metric** breaching some preset **condition**.
+Notifications **chanels** set here will be **email** and +[**ntfy**](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal) + +![alert](https://i.imgur.com/b4hchSu.png) + +## The setup + +To **add** alertmanager to the current stack: + +* **New file** - `alertmanager.yml` will be **bind mounted** in alertmanager container.
+ This is the **configuration** on how and where **to deliver** alerts.
+ +
+ alertmanager.yml + + ```yml + route: + receiver: 'email' + + receivers: + - name: 'ntfy' + webhook_configs: + - url: 'https://ntfy.example.com/alertmanager' + send_resolved: true + + - name: 'email' + email_configs: + - to: 'whoever@example.com' + from: 'alertmanager@example.com' + smarthost: smtp-relay.sendinblue.com:587 + auth_username: '' + auth_identity: '' + auth_password: '' + ``` +
+ +* **New file** - `alert.rules` will be **bind mounted** in to prometheus container
+ This file **defines** at what value a metric becomes an **alert** event. + +
+ alert.rules + + ```yml + groups: + - name: host + rules: + - alert: DiskSpaceLow + expr: sum(node_filesystem_free_bytes{fstype="ext4"}) > 19 + for: 10s + labels: + severity: critical + annotations: + description: "Diskspace is low!" + ``` +
+ +* **Changed** `prometheus.yml`. Added **alerting section** that points to alertmanager + container, and also **set path** to a `rules` file. + +
+ prometheus.yml + + ```yml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'nodeexporter' + static_configs: + - targets: ['nodeexporter:9100'] + + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - 'alertmanager:9093' + + rule_files: + - '/etc/prometheus/rules/alert.rules' + ``` +
+ +* **New container** - `alertmanager` added to the compose file and **prometheus + container** has bind mount **rules file** added. + +
+ docker-compose.yml + + ```yml + services: + + # MONITORING SYSTEM AND THE METRICS DATABASE + prometheus: + image: prom/prometheus:v2.42.0 + container_name: prometheus + hostname: prometheus + restart: unless-stopped + user: root + depends_on: + - cadvisor + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=240h' + - '--web.enable-lifecycle' + volumes: + - ./prometheus_data:/prometheus + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./alert.rules:/etc/prometheus/rules/alert.rules + expose: + - "9090" + labels: + org.label-schema.group: "monitoring" + + # ALERT MANAGMENT FOR PROMETHEUS + alertmanager: + image: prom/alertmanager:v0.25.0 + container_name: alertmanager + hostname: alertmanager + restart: unless-stopped + volumes: + - ./alertmanager.yml:/etc/alertmanager.yml + - ./alertmanager_data:/alertmanager + command: + - '--config.file=/etc/alertmanager.yml' + - '--storage.path=/alertmanager' + expose: + - "9093" + labels: + org.label-schema.group: "monitoring" + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +* **Adding** alertmanager to the **Caddyfile** of the reverse proxy so that + it can be reached at `https://alert.example.com`. **Not necessary**, + but useful as it **allows to send alerts from anywhere**, + not just from prometheus, or other containers on same docker network. + +
+ Caddyfile + + ```php + alert.{$MY_DOMAIN} { + reverse_proxy alertmanager:9093 + } + ``` +
+ +## The basics + +![alert](https://i.imgur.com/C7g0xJt.png) + + +Once above setup is done **an alert** about low disk space **should fire** +and a **notification** email should come.
+In `alertmanager.yml` switch from email **to ntfy** can be done. + +*Useful* + +* **alert** from anywhere using **curl**:
+ `curl -H 'Content-Type: application/json' -d '[{"labels":{"alertname":"blabla"}}]' https://alert.example.com/api/v1/alerts` +* **reload rules**:
+ `curl -X POST https://prom.example.com/-/reload` + +[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom#define-alerts) +has more detailed section on alerting worth checking out. + +# Loki + +![loki_arch](https://i.imgur.com/aoMPrVV.png) + +Loki is made by the grafana team. Sometimes called a Prometheus for logs, +it's a **push** type monitoring, where an agent - **promtail** +scrapes logs and then pushes them on to a Loki instance.
+ +For docker containers theres also an option to install +[**loki-docker-driver**](https://grafana.com/docs/loki/latest/clients/docker-driver/) +on a docker host and log pushing is set either globally in /etc/docker/daemon.json +or per container in compose files.
+But as it turns out, **promtail** capabilities might be missed, +like its ability to add labels to logs it scrapes based on some rule. +Or processing data in some way, like translate IP addresses in to country +names or cities.
+Still loki-docker-driver is useful for getting containers logs in to loki +quickly and easily, with less cluttering of compose and less containers runnig. + +There will be **two examples** of logs monitoring.
+A **minecraft server** and a **caddy revers proxy**, both docker containers. + +## Loki setup + +* **New container** - `loki` added to the compose file.
+ Note the port 3100 is actually mapped to the host, + allowing `localhost:3100` from driver to work. + +
+ docker-compose.yml + + ```yml + services: + + # LOG MANAGMENT WITH LOKI + loki: + image: grafana/loki:2.7.3 + container_name: loki + hostname: loki + user: root + restart: unless-stopped + volumes: + - ./loki_data:/loki + - ./loki-config.yml:/etc/loki-config.yml + command: + - '-config.file=/etc/loki-config.yml' + ports: + - "3100:3100" + labels: + org.label-schema.group: "monitoring" + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +* **New file** - `loki-config.yml` bind mounted in the loki container.
+ The config comes from + [the official example](https://github.com/grafana/loki/tree/main/cmd/loki), + with some changes. + * **URL** changed for this setup. + * **Compactor** section is added, to have control over + [data retention.](https://grafana.com/docs/loki/latest/operations/storage/retention/) + * **Fixing** error - *"too many outstanding requests"*, discussion + [here.](https://github.com/grafana/loki/issues/5123)
+ It turns off parallelism, both split by time interval and shards split. + +
+ loki-config.yml + + ```yml + auth_enabled: false + + server: + http_listen_port: 3100 + + common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + # --- disable splitting to fix "too many outstanding requests" + + query_range: + parallelise_shardable_queries: false + + # --- compactor to have control over length of data retention + + compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + + limits_config: + retention_period: 240h + split_queries_by_interval: 0 # part of disable splitting fix + + # ------------------------------------------------------- + + schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + + ruler: + alertmanager_url: http://alertmanager:9093 + + analytics: + reporting_enabled: false + ``` +
+ +* #### loki-docker-driverdriver + + * **Install** [loki-docker-driver](https://grafana.com/docs/loki/latest/clients/docker-driver/)
+ `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions`
+ To check if it's installed and enabled: `docker plugin ls`
+ * Containers that should be monitored usind loki-docker-driver need + `logging` section in their compose. + +
+ docker-compose.yml + + ```yml + services: + + whoami: + image: "containous/whoami" + container_name: "whoami" + hostname: "whoami" + logging: + driver: "loki" + options: + loki-url: "http://localhost:3100/loki/api/v1/push" + ``` +
+ +* #### promtail + + * Containers that should be monitored with **promtail** need it **added** + **to** their **compose** file, and made sure that it has access to the log files. + +
+ minecraft-docker-compose.yml + + ```yml + services: + + minecraft: + image: itzg/minecraft-server + container_name: minecraft + hostname: minecraft + restart: unless-stopped + env_file: .env + tty: true + stdin_open: true + ports: + - 25565:25565 # minecraft server players connect + volumes: + - ./minecraft_data:/data + + # LOG AGENT PUSHING LOGS TO LOKI + promtail: + image: grafana/promtail + container_name: minecraft-promtail + hostname: minecraft-promtail + restart: unless-stopped + volumes: + - ./minecraft_data/logs:/var/log/minecraft:ro + - ./promtail-config.yml:/etc/promtail-config.yml + command: + - '-config.file=/etc/promtail-config.yml' + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +
+ caddy-docker-compose.yml + + ```yml + services: + + caddy: + image: caddy + container_name: caddy + hostname: caddy + restart: unless-stopped + env_file: .env + ports: + - "80:80" + - "443:443" + - "443:443/udp" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ./caddy_config:/data + - ./caddy_data:/config + - ./caddy_logs:/var/log/caddy + + # LOG AGENT PUSHING LOGS TO LOKI + promtail: + image: grafana/promtail + container_name: caddy-promtail + hostname: caddy-promtail + restart: unless-stopped + volumes: + - ./caddy_logs:/var/log/caddy:ro + - ./promtail-config.yml:/etc/promtail-config.yml + command: + - '-config.file=/etc/promtail-config.yml' + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + + ``` +
+ + * Generic **config file for promtail**, needs to be bind mounted + +
+ promtail-config.yml + + ```yml + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: blablabla + static_configs: + - targets: + - localhost + labels: + job: blablabla_log + __path__: /var/log/blablabla/*.log + ``` +
+ +# Minecraft Loki example + +What can be seen in this example: + +* How to monitor logs of a docker container, + a [minecraft server](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/minecraft). +* How to visualize the logs in a dashboard. +* How to set an alert when a specific pattern appears in the logs. +* How to extract information from log to include it in the alert notification. +* Basic of grafana alert templates, so that the notification actually looks good, + and shows only relevant info. + +**Requirements** - grafana, loki, minecraft. + +![logo-minecraft](https://i.imgur.com/VphJTKG.png) + +### The Setup + +Initially **loki-docker-driver was used** to get logs to Loki, and it was simple +and worked nicely. But during alert stage +I **could not** figure out how to extract string from logs and include it in +an **alert** notification. Specificly to not just say that "a player joined", +but to have there name of the player that joined.
+The way to solve that was to **switch to promtail** and make use of its +[pipeline_stages](https://grafana.com/docs/loki/latest/clients/promtail/pipelines/). +Which was **suprisingly simple** and elegant. + +**Promtail** container is added to minecraft's **compose**, with bind mount +access to minecraf's logs.
+ +
+minecraft-docker-compose.yml + +```yml +services: + + minecraft: + image: itzg/minecraft-server + container_name: minecraft + hostname: minecraft + restart: unless-stopped + env_file: .env + tty: true + stdin_open: true + ports: + - 25565:25565 # minecraft server players connect + volumes: + - ./minecraft_data:/data + + # LOG AGENT PUSHING LOGS TO LOKI + promtail: + image: grafana/promtail + container_name: minecraft-promtail + hostname: minecraft-promtail + restart: unless-stopped + volumes: + - ./minecraft_data/logs:/var/log/minecraft:ro + - ./promtail-config.yml:/etc/promtail-config.yml + command: + - '-config.file=/etc/promtail-config.yml' + +networks: + default: + name: $DOCKER_MY_NETWORK + external: true +``` +
+ +**Promtail's config** is similar to the generic config in the previous section.
+The only addition is a short **pipeline** stage with a **regex** that runs against +every log line before sending it to Loki. When a line matches, **a label** `player` +is added to that log line. +The value of that label comes from the **named capture group** thats part of +that regex, the [syntax](https://www.regular-expressions.info/named.html) +is: `(?Pgroup)`
+This label will be easy to use later in the alert stage. + +
+promtail-config.yml + +```yml +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: minecraft + static_configs: + - targets: + - localhost + labels: + job: minecraft_logs + __path__: /var/log/minecraft/*.log + pipeline_stages: + - regex: + expression: '.*:\s(?P.*)\sjoined the game$' + - labels: + player: +``` +
+ +[Here's regex101](https://regex101.com/r/5vkOU2/1) of it, +with some data to show how it works and bit of explanation.
+[Here's](https://stackoverflow.com/a/74962269/1383369) +the stackoverflow answer that is the source for that config. + +![regex](https://i.imgur.com/bT5XSHn.png) + +### First steps in Grafana + +* In **grafana**, loki needs to be added as a **datasource**, `http://loki:3100` +* In **Explore section**, filter, job = `minecraft_logs`, **Run query** button... + this should result in seeing minecraft logs and their volume/time graph. + +This Explore view will be recreated as a dashboard. + +### Dashboard minecraft_logs + +![dashboard-minecraft](https://i.imgur.com/M1k0Dn4.png) + +* New dashboard, new panel + * Data source - Loki + * Switch from `builder` to `code`
+ * query - `count_over_time({container_name="minecraft"} |= `` [1m])`
+ * Transform - Rename by regex - `(.*)` - `Logs` + * Graph type - `Time series` + * Title - Logs volume + * Transparent background + * Legend off + * Graph styles - bar + * Fill opacity - 50 + * Color scheme - single color + * `Query options` - Min interval=1m + * Save +* Add another pane to the dashboard + * Graph type - `Logs` + * Data source - Loki + * Switch from `builder` to `code`
+ query - `{container_name="minecraft"} |= ""`
+ * Title - *empty* + * Deduplication - Signature + * Save + +This should create a similar dashboard to the one in the picture above.
+ +[Performance tips](https://www.youtube.com/watch?v=YED8XIm0YPs) +for grafana loki queries + +## Alerts in Grafana for Loki + +![alert-labels](https://i.imgur.com/LuUBZFn.png) + +When **a player joins** minecraft server a **log appears** *"Bastard joined the game"*
+**Alert will be set** to look for string *"joined the game"* and **send notification** +when it occurs. + +Now, might be good time to **brush up on PromQL / LogQL** and the **data types** +they return when a query happens. That **instant vector** and **range vector** +thingie. As grafana will scream when using range vector. + +### Create alert rule + +- **1 Set an alert rule name** + - Rule name = Minecraft-player-joined-alert +- **2 Set a query and alert condition** + - **A** - Switch to Loki; set Last 5 minutes + - switch from builder to code + - `count_over_time({job="minecraft_logs"} |= "joined the game" [5m])` + - **B** - Reduce + - Function = Last + - Input = A + - Mode = Strict + - **C** - Treshold + - Input = B + - is above 0 + - Make this the alert condition +- **3 Alert evaluation behavior** + - Folder = "Alerts" + - Evaluation group (interval) = "five-min"
+ - Evaluation interval = 5m + - For 0s + - Configure no data and error handling + - Alert state if no data or all values are null = OK +- **4 Add details for your alert rule** + - Here is where the label `player` that was set in **promtail** is used
+ Summary = `{{ $labels.player }} joined the Minecraft server.` + - Can also pass values from expressions by targeting A/B/C/.. from step2
+ Description = `Number of players that joined in the last 5 min: {{ $values.B }}`
+- **5 Notifications** + - nothing +- Save and exit + +### Contact points + + - New contact point + - Name = ntfy + - Integration = Webhook + - URL = https://ntfy.example.com/grafana + - Title = `{{ .CommonAnnotations.summary }}` + - Message = I put in [empty space unicode character](https://emptycharacter.com/) + - Disable resolved message = check + - Test + - Save + +### Notification policies + + - Edit default + - Default contact point = ntfy + - Save + +After all this, there should be notification coming when a player joins. + +### grafana-to-ntfy + +For **alerts** one can use +[ntfy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal) +but on its own alerts from grafana are **just plain text json**.
+[Here's](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal#grafana-to-ntfy) +how to setup grafana-to-ntfy, to **make alerts look good**. + +![ntfy](https://i.imgur.com/gL81jRg.png) + +--- +--- + +### Templates + +Not really used here, but **they are pain** in the ass and there's some info +as it took **embarrassingly** long to find that +`{{ .CommonAnnotations.summary }}` for the title. + +* **Testing** should be done in **contact point** when editing, + useful **Test button** that allows you send alerts with custom values. +* To [define a template.](https://i.imgur.com/ZczwCx2.png) +* To [call a template.](https://i.imgur.com/0YdWA8Q.png) +* My **big mistake** when playing with this was **missing a dot**.
+ In Contact point, in Title/Message input box. + * correct one - `{{ template "test" . }}` + * the one I had - `{{ template "test" }}`
+* So yeah, **dot** is important in here. It represents **data and context** + passed to a template. It can represent **global context**, or when used inside + `{{ range }}` it represents **iteration** loop value. +* [This](https://pastebin.com/id3264k6) json structure is what an **alert** looks + like. Notice `alerts` being an **array** and `commonAnnotations` being **object**. + For **array** theres need to **loop** over it to get access to the + values in it. For **objects** one just needs to target **the name** + from global context.. **using dot** at the beginning. +* To [iterate over alerts array.](https://i.imgur.com/gdwGhjN.png) +* To just access a value - `{{ .CommonAnnotations.summary }}` +* Then theres **conditional** things one can do in **golang templates**, + but I am not going to dig that deep... + +Templates resources + +* [Overview of Grafana Alerting and Message Templating for Slack](https://faun.pub/overview-of-grafana-alerting-and-message-templating-for-slack-6bb740ec44af) +* [youtube - Unified Alerting Grafana 8 | Prometheus | Victoria | Telegraf | Notifications | Alert Templating](https://youtu.be/UtmmhLraSnE) +* [Dot notation](https://www.practical-go-lessons.com/chap-32-templates#dot-notation) + +--- +--- + +# Caddy reverse proxy monitoring + +What can be seen in this example: + +* Use of **Prometheus** to **monitor** a docker **container** - + [caddy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy). +* How to **import a dashobard** to grafana. +* Use of **Loki** to monitor **logs** of a docker **container**. +* How to set **Promtail** to push only certain values and label them. +* Create **dashboard** in grafana from data in **Loki**. + +**Requirements** - grafana, loki, caddy. + +![logo-minecraft](https://i.imgur.com/HU4kHCj.png) + +**Reverse proxy** is kinda linchpin of a selfhosted setup as it is **in charge** +of all the http/https **traffic** that goes in. So focus on monitoring this +**keystone** makes sense. + +**Requirements** - grafana, prometheus, loki, caddy container + +## Caddy - Metrics - Prometheus + +![logo](https://i.imgur.com/6QdZuVR.png) + +**Caddy** has build in **exporter** of metrics for prometheus, so all that is needed +is enabling it, **scrape it** by prometheus, and import a **dashboard**. + +* Edit Caddyfile to [enable metrics.](https://caddyserver.com/docs/metrics) + +
+ Caddyfile + + ```php + { + servers { + metrics + } + + admin 0.0.0.0:2019 + } + + + a.{$MY_DOMAIN} { + reverse_proxy whoami:80 + } + ``` +
+ +* Edit compose to publish 2019 port.
+ Likely **not necessary** if Caddy and Prometheus are on the **same docker network**, + but its nice to check if the metrics export works at `:2019/metrics` + +
+ docker-compose.yml + + ```yml + services: + + caddy: + image: caddy + container_name: caddy + hostname: caddy + restart: unless-stopped + env_file: .env + ports: + - "80:80" + - "443:443" + - "443:443/udp" + - "2019:2019" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ./caddy_config:/data + - ./caddy_data:/config + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +* Edit **prometheus.yml** to add caddy **scraping** point + +
+ prometheus.yml + + ```yml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'caddy' + static_configs: + - targets: ['caddy:2019'] + ``` +
+ +* In grafana **import** + [caddy dashboard](https://grafana.com/grafana/dashboards/14280-caddy-exporter/)
+ or make your own, `caddy_reverse_proxy_upstreams_healthy` shows reverse proxy + upstreams, but thats all. + +But these **metrics** are more about **performance** and **load** put on Caddy, +which in selfhosted environment will **likely be minimal** and not interesting.
+To get **more intriguing** info of who, when, **from where**, connects +to what **service**,.. well for that monitoring of **access logs** is needed. + +--- +--- + +## Caddy - Logs - Loki + +![logs_dash](https://i.imgur.com/j9CcJ44.png) + +**Loki** itself just **stores** the logs. To get them to Loki a **Promtail** container is used +that has **access** to caddy's **logs**. Its job is to **scrape** them regularly, maybe +**process** them in some way, and then **push** them to Loki.
+Once there, a basic grafana **dashboard** can be made. + + +### The setup + +* Have Grafana, Loki, Caddy working +* Edit Caddy **compose**, bind mount `/var/log/caddy`.
+ **Add** to the compose also **Promtail container**, that has the same logs bind mount, + along with bind mount of its **config file**.
+ Promtail will scrape logs to which it now has access and **pushes** them **to Loki.** + +
+ docker-compose.yml + + ```yml + services: + + caddy: + image: caddy + container_name: caddy + hostname: caddy + restart: unless-stopped + env_file: .env + ports: + - "80:80" + - "443:443" + - "443:443/udp" + - "2019:2019" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ./caddy_data:/data + - ./caddy_config:/config + - ./caddy_logs:/var/log/caddy + + # LOG AGENT PUSHING LOGS TO LOKI + promtail: + image: grafana/promtail + container_name: caddy-promtail + hostname: caddy-promtail + restart: unless-stopped + volumes: + - ./promtail-config.yml:/etc/promtail-config.yml + - ./caddy_logs:/var/log/caddy:ro + command: + - '-config.file=/etc/promtail-config.yml' + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +
+ promtail-config.yml + + ```yml + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: caddy_access_log + static_configs: + - targets: + - localhost + labels: + job: caddy_access_log + host: example.com + agent: caddy-promtail + __path__: /var/log/caddy/*.log + ``` +
+ +* **Promtail** scrapes a logs, **one line** at the time and is able to do **neat + things** with it before sending it - add labels, ignore entire line if some condition is met, + only send useful values,...
+ [Pipelines](https://grafana.com/docs/loki/latest/clients/promtail/pipelines/) + are used for this. + The **example here** is pretty straight forward, in **json** stage a desired + value is **extracted**, **template** is created with the use of that value, + and **output** is set. + [Here's](https://zerokspot.com/weblog/2023/01/25/testing-promtail-pipelines/) + some more to read with an example. + +
+ promtail-config.yml customizing fields + + ```yml + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: caddy_access_log + + static_configs: + - targets: + - localhost + labels: + job: caddy_access_log + host: example.com + agent: caddy-promtail + __path__: /var/log/caddy/*.log + + pipeline_stages: + - json: + expressions: + request_remote_ip: request.remote_ip + + - template: + source: output # creates empty output variable + template: '{"remote_ip": {{.request_remote_ip}}}' + + - output: + source: output + + ``` +
+ +* Edit `Caddyfile` to enable + [**access logs**](https://caddyserver.com/docs/caddyfile/directives/log). + Unfortunetly this **can't be globally** enabled, so the easiest way seems to be + to create a **logging** [**snippet**](https://caddyserver.com/docs/caddyfile/concepts#snippets) + called `log_common` and copy paste the **import line** in to every site block. + +
+ Caddyfile + + ```yml + (log_common) { + log { + output file /var/log/caddy/caddy_access.log + } + } + + ntfy.example.com { + import log_common + reverse_proxy ntfy:80 + } + + mealie.{$MY_DOMAIN} { + import log_common + reverse_proxy mealie:80 + } + ``` +
+ +* at this points logs should be visible and **explorable in grafana**
+ Explore > `{job="caddy_access_log"} |= "" | json` + +### Geoip + +![geoip_info](https://i.imgur.com/f4P8ydl.png) + +**Promtail** got recently a **geoip stage**. One can feed it an **IP address** and an mmdb **geoIP +datbase** and it adds geoip **labels** to the log entry. + +[The official documentation.](https://github.com/grafana/loki/blob/main/docs/sources/clients/promtail/stages/geoip.md) + +* **Register** a free account on [maxmind.com](https://www.maxmind.com/en/geolite2/signup). +* **Download** one of the mmdb format **databases** + * `GeoLite2 City` - 70MB full geoip info - city, postal code, time zone, latitude/longitude,.. + * `GeoLite2 Country` 6MB, just country and continent +* **Bind mount** whichever database in to **promtail container**. + +
+ docker-compose.yml + + ```yml + services: + + caddy: + image: caddy + container_name: caddy + hostname: caddy + restart: unless-stopped + env_file: .env + ports: + - "80:80" + - "443:443" + - "443:443/udp" + - "2019:2019" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - ./caddy_data:/data + - ./caddy_config:/config + - ./caddy_logs:/var/log/caddy + + # LOG AGENT PUSHING LOGS TO LOKI + promtail: + image: grafana/promtail + container_name: caddy-promtail + hostname: caddy-promtail + restart: unless-stopped + volumes: + - ./promtail-config.yml:/etc/promtail-config.yml + - ./caddy_logs:/var/log/caddy:ro + - ./GeoLite2-City.mmdb:/etc/GeoLite2-City.mmdb:ro + command: + - '-config.file=/etc/promtail-config.yml' + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` + +* In **promtail** config, **json stage** is added where IP address is loaded in to + a **variable** called `remote_ip`, which then is used in **geoip stage**. + If all else is set correctly, the geoip **labels** are automaticly added to the log entry. + +
+ geoip promtail-config.yml + + ```yml + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: caddy_access_log + + static_configs: + - targets: + - localhost + labels: + job: caddy_access_log + host: example.com + agent: caddy-promtail + __path__: /var/log/caddy/*.log + + pipeline_stages: + - json: + expressions: + remote_ip: request.remote_ip + + - geoip: + db: "/etc/GeoLite2-City.mmdb" + source: remote_ip + db_type: "city" + ``` +
+ +Can be tested with opera build in VPN, or some online +[site tester](https://pagespeed.web.dev/). + +### Dashboard + +![pane1](https://i.imgur.com/hW92sLO.png) + +* **new pane**, will be **time series** graph showing **Subdomains hits timeline** + + * Graph type = Time series + * Data source = Loki + * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [1m])) by (request_host)` + * Query options > Min interval = 1m + * Transform > Rename by regex + * Match = `\{request_host="(.*)"\}` + * Replace = `$1` + * Title = "Subdomains hits timeline" + * Transparent + * Tooltip mode = All + * Tooltip values sort order = Descending + * Legen Placement = Right + * Value = Total + * Graph style = Bars + * Fill opacity = 50 + +![pane2](https://i.imgur.com/KYZdotg.png) + +* Add **another pane**, will be a **pie chart**, showing **subdomains** divide + + * Graph type = Pie chart + * Data source = Loki + * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [$__range])) by (request_host)` + * Query options > Min interval = 1m + * Transform > Rename by regex + * Match = `\{request_host="(.*)"\}` + * Replace = `$1` + * Title = "Subdomains use" + * Transparent + * Legen Placement = Right + * Value = Last + +![pane3](https://i.imgur.com/MjbLVlJ.png) + +* Add **another pane**, will be a **Geomap**, showing location of machine accessing + Caddy + + * Graph type = Geomap + * Data source = Loki + * switch from builder to code
+ `{job="caddy_access_log"} |= "" | json` + * Query options > Min interval = 1m + * Transform > Extract fields + * Source = labels + * Format = JSON + * 1. Field = geoip_location_latitude; Alias = latitude + * 2. Field = geoip_location_longitude; Alias = longitude + * Title = "Geomap" + * Transparent + * Map view > View > *Drag and zoom around* > Use current map setting + +* Add **another pane**, will be a **pie chart**, showing **IPs** that hit the most + + * Graph type = Pie chart + * Data source = Loki + * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [$__range])) by (request_remote_ip)` + * Query options > Min interval = 1m + * Transform > Rename by regex + * Match = `\{request_remote_ip="(.*)"\}` + * Replace = `$1` + * Title = "IPs by number of requests" + * Transparent + * Legen Placement = Right + * Value = Last or Total + +* Add **another pane**, this will be actual **log view** + + * Graph type - Logs + * Data source - Loki + * Switch from builder to code + * query - `{job="caddy_access_log"} |= "" | json` + * Title - empty + * Deduplication - Exact or Signature + * Save + +![pane3](https://i.imgur.com/bzE6JEg.png) + + +# Update + +Manual image update: + +- `docker-compose pull`
+- `docker-compose up -d`
+- `docker image prune` + +# Backup and restore + +#### Backup + +Using [borg](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/borg_backup) +that makes daily snapshot of the entire directory. + +#### Restore + +* down the prometheus containers `docker-compose down`
+* delete the entire prometheus directory
+* from the backup copy back the prometheus directory
+* start the containers `docker-compose up -d`