diff --git a/prometheus_grafana_loki/dashboards/docker_containers.json b/prometheus_grafana_loki/dashboards/docker_containers.json
new file mode 100644
index 0000000..9589e7c
--- /dev/null
+++ b/prometheus_grafana_loki/dashboards/docker_containers.json
@@ -0,0 +1,1270 @@
+{
+ "id": null,
+ "title": "Docker Containers",
+ "description": "Containers metrics",
+ "tags": [
+ "docker"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": true,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "150px",
+ "panels": [
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": true,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 4,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\"}[1m])) / count(node_cpu_seconds_total{mode=\"user\"}) * 100",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": "65, 90",
+ "title": "CPU Load",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 7,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "machine_cpu_cores",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "metric": "machine_cpu_cores",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": "",
+ "title": "CPU Cores",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": true,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 5,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "(sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100",
+ "interval": "10s",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": "65, 90",
+ "title": "Memory Load",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 2,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "sum(container_memory_usage_bytes{image!=\"\"})",
+ "interval": "10s",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "10s",
+ "title": "Used Memory",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(50, 172, 45, 0.97)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(245, 54, 54, 0.9)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": null,
+ "editable": true,
+ "error": false,
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": true,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 6,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "(node_filesystem_size_bytes{fstype=\"ext4\"} - node_filesystem_free_bytes{fstype=\"ext4\"}) / node_filesystem_size_bytes{fstype=\"ext4\"} * 100",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 30
+ }
+ ],
+ "thresholds": "65, 90",
+ "title": "Storage Load",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 3,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "sum(container_fs_usage_bytes)",
+ "interval": "30s",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": "",
+ "title": "Used Storage",
+ "transparent": false,
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ }
+ ],
+ "title": "Overview"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "150px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": true,
+ "datasource": "Prometheus",
+ "decimals": 0,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)",
+ "thresholdLine": false
+ },
+ "id": 9,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "scalar(count(container_memory_usage_bytes{image!=\"\"}) > 0)",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "containers",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Running Containers",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 10,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "load 1m",
+ "color": "#BF1B00"
+ }
+ ],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_load1",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "load 1m",
+ "metric": "node_load1",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "System Load",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 15,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "read",
+ "yaxis": 1
+ },
+ {
+ "alias": "written",
+ "yaxis": 1
+ },
+ {
+ "alias": "io time",
+ "yaxis": 2
+ }
+ ],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(irate(node_disk_read_bytes_total[5m]))",
+ "interval": "2s",
+ "intervalFactor": 4,
+ "legendFormat": "read",
+ "metric": "",
+ "refId": "A",
+ "step": 8
+ },
+ {
+ "expr": "sum(irate(node_disk_written_bytes_total[5m]))",
+ "interval": "2s",
+ "intervalFactor": 4,
+ "legendFormat": "written",
+ "metric": "",
+ "refId": "B",
+ "step": 8
+ },
+ {
+ "expr": "sum(irate(node_disk_io_time_seconds_total[5m]))",
+ "interval": "2s",
+ "intervalFactor": 4,
+ "legendFormat": "io time",
+ "metric": "",
+ "refId": "C",
+ "step": 8
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "I/O Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "title": "Host stats"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 8,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m])) / scalar(count(node_cpu_seconds_total{mode=\"user\"})) * 100",
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "metric": "container_cpu_user_seconds_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container CPU Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "CPU"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 11,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name)(container_memory_usage_bytes{image!=\"\",container_label_org_label_schema_group=\"\"})",
+ "intervalFactor": 1,
+ "legendFormat": "{{ name }}",
+ "metric": "container_memory_usage",
+ "refId": "A",
+ "step": 1
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container Memory Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 12,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (container_memory_cache{image!=\"\",container_label_org_label_schema_group=\"\"})",
+ "intervalFactor": 2,
+ "legendFormat": "{{name}}",
+ "metric": "container_memory_cache",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container Cached Memory Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "Memory"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 13,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(container_network_receive_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))",
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "metric": "container_network_receive_bytes_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container Network Input",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 14,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))",
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "metric": "container_network_transmit_bytes_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container Network Output",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "Network"
+ }
+ ],
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": "10s",
+ "schemaVersion": 12,
+ "version": 8,
+ "links": [],
+ "gnetId": null
+}
diff --git a/prometheus_grafana_loki/dashboards/docker_host.json b/prometheus_grafana_loki/dashboards/docker_host.json
new file mode 100644
index 0000000..a6beef7
--- /dev/null
+++ b/prometheus_grafana_loki/dashboards/docker_host.json
@@ -0,0 +1,1441 @@
+{
+ "id": null,
+ "title": "Docker Host",
+ "description": "Docker host metrics",
+ "tags": [
+ "system"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": true,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "100px",
+ "panels": [
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 1,
+ "editable": true,
+ "error": false,
+ "format": "s",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 1,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "s",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "node_time_seconds - node_boot_time_seconds",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A",
+ "step": 30
+ }
+ ],
+ "thresholds": "",
+ "title": "Uptime",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "percent",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 13,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))",
+ "interval": "10s",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": "",
+ "title": "CPU Idle",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 12,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "machine_cpu_cores",
+ "intervalFactor": 2,
+ "metric": "machine_cpu_cores",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": "",
+ "title": "CPU Cores",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 2,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "node_memory_MemAvailable_bytes",
+ "interval": "30s",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": "",
+ "title": "Available Memory",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 3,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "node_memory_SwapFree_bytes",
+ "interval": "30s",
+ "intervalFactor": 2,
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "thresholds": "",
+ "title": "Free Swap",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "id": 4,
+ "interval": null,
+ "isNew": true,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "span": 2,
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "targets": [
+ {
+ "expr": "sum(node_filesystem_free_bytes{fstype=\"ext4\"})",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 30
+ }
+ ],
+ "thresholds": "",
+ "title": "Free Storage",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg",
+ "timeFrom": "10s",
+ "hideTimeOverride": true
+ }
+ ],
+ "title": "Available resources"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "150px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": true,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 9,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "load 1m",
+ "color": "#1F78C1"
+ }
+ ],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_load1",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "load 1m",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Load Average 1m",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 10,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "blocked by I/O",
+ "color": "#58140C"
+ }
+ ],
+ "span": 4,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_procs_running",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "running",
+ "metric": "node_procs_running",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "node_procs_blocked",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "blocked by I/O",
+ "metric": "node_procs_blocked",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Processes",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 11,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "interrupts",
+ "color": "#806EB7"
+ }
+ ],
+ "span": 4,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": " irate(node_intr_total[5m])",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "interrupts",
+ "metric": "node_intr_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Interrupts",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "Load"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 4,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 5,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(node_cpu_seconds_total[1m])) by (mode) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"}))",
+ "intervalFactor": 10,
+ "legendFormat": "{{ mode }}",
+ "metric": "node_cpu_seconds_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "CPU Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": 100,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "title": "CPU"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 4,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 6,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Used",
+ "color": "#BF1B00"
+ },
+ {
+ "alias": "Free",
+ "color": "#7EB26D"
+ },
+ {
+ "alias": "Buffers",
+ "color": "#6ED0E0"
+ },
+ {
+ "alias": "Cached",
+ "color": "#EF843C"
+ }
+ ],
+ "span": 12,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "refId": "A",
+ "step": 1
+ },
+ {
+ "expr": "node_memory_MemFree_bytes",
+ "intervalFactor": 1,
+ "legendFormat": "Free",
+ "refId": "B",
+ "step": 1
+ },
+ {
+ "expr": "node_memory_Buffers_bytes",
+ "intervalFactor": 1,
+ "legendFormat": "Buffers",
+ "refId": "C",
+ "step": 1
+ },
+ {
+ "expr": "node_memory_Cached_bytes",
+ "intervalFactor": 1,
+ "legendFormat": "Cached",
+ "refId": "D",
+ "step": 1
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "title": "Memory"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 7,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "read",
+ "yaxis": 1
+ },
+ {
+ "alias": "written",
+ "yaxis": 1
+ },
+ {
+ "alias": "io time",
+ "yaxis": 2
+ }
+ ],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(irate(node_disk_read_bytes_total[1m]))",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "metric": "node_disk_read_bytes_total",
+ "refId": "A",
+ "step": 1
+ },
+ {
+ "expr": "sum(irate(node_disk_written_bytes_total[1m]))",
+ "intervalFactor": 1,
+ "legendFormat": "written",
+ "metric": "node_disk_written_bytes_total",
+ "refId": "B",
+ "step": 1
+ },
+ {
+ "expr": "sum(irate(node_disk_io_time_seconds_total[1m]))",
+ "intervalFactor": 1,
+ "legendFormat": "io time",
+ "metric": "node_disk_io_time_seconds_total",
+ "refId": "C",
+ "step": 1
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "I/O Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "ms",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "title": "I/O"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 4,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 8,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))",
+ "intervalFactor": 1,
+ "legendFormat": "In: {{ device }}",
+ "metric": "node_network_receive_bytes_total",
+ "refId": "A",
+ "step": 1
+ },
+ {
+ "expr": "sum(rate(container_network_transmit_bytes_total{id=\"/\",interface!~\"br.*|lo\"}[1m]))",
+ "intervalFactor": 1,
+ "legendFormat": "Out: {{ device }}",
+ "metric": "node_network_transmit_bytes_total",
+ "refId": "B",
+ "step": 1
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Network Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "Network"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 4,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 14,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Used",
+ "color": "#890F02"
+ },
+ {
+ "alias": "Free",
+ "color": "#7EB26D"
+ }
+ ],
+ "span": 6,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "Used",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "node_memory_SwapFree_bytes",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "Free",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Swap Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 15,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(node_vmstat_pswpin[1m]) * 4096 or irate(node_vmstat_pswpin[5m]) * 4096",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "In",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "rate(node_vmstat_pswpout[1m]) * 4096 or irate(node_vmstat_pswpout[5m]) * 4096",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "Out",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Swap I/O",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "show": true
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": 0,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ]
+ }
+ ],
+ "title": "New row"
+ }
+ ],
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": "10s",
+ "schemaVersion": 12,
+ "version": 2,
+ "links": [],
+ "gnetId": null
+}
diff --git a/prometheus_grafana_loki/dashboards/monitoring_services.json b/prometheus_grafana_loki/dashboards/monitoring_services.json
new file mode 100644
index 0000000..00df64a
--- /dev/null
+++ b/prometheus_grafana_loki/dashboards/monitoring_services.json
@@ -0,0 +1,3412 @@
+{
+ "id": null,
+ "title": "Monitoring Services",
+ "tags": [
+ "prometheus"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": true,
+ "panels": [
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 1,
+ "editable": true,
+ "error": false,
+ "format": "s",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "hideTimeOverride": true,
+ "id": 1,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "s",
+ "postfixFontSize": "80%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "(time() - process_start_time_seconds{instance=\"localhost:9090\",job=\"prometheus\"})",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "10s",
+ "timeShift": null,
+ "title": "Prometheus Uptime",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "format": "bytes",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "hideTimeOverride": true,
+ "id": 5,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "sum(container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})",
+ "format": "time_series",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "10s",
+ "timeShift": null,
+ "title": "Memory Usage",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "hideTimeOverride": true,
+ "id": 3,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_chunks",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "10s",
+ "timeShift": null,
+ "title": "In-Memory Chunks",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "hideTimeOverride": true,
+ "id": 2,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "",
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_series",
+ "interval": "10s",
+ "intervalFactor": 1,
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": "",
+ "timeFrom": "10s",
+ "timeShift": null,
+ "title": "In-Memory Series",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 3
+ },
+ "id": 6,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu{mode=\"user\"})))",
+ "format": "time_series",
+ "hide": true,
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "sum by (name) (rate(container_cpu_user_seconds_total{container_label_org_label_schema_group=\"monitoring\"}[1m]) * 100 / scalar(count(node_cpu_seconds_total{mode=\"user\"})))",
+ "format": "time_series",
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container CPU Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "decimals": 2,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 7,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": true,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum by (name) (container_memory_usage_bytes{container_label_org_label_schema_group=\"monitoring\"})",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 10,
+ "legendFormat": "{{ name }}",
+ "metric": "container_memory_usage_bytes",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Container Memory Usage",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "decimals": 0,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 73,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": false,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": true,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(ALERTS{alertstate=\"firing\"}) by (alertname)",
+ "format": "time_series",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "legendFormat": "{{ alertname }}",
+ "metric": "container_memory_usage_bytes",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Alerts",
+ "tooltip": {
+ "msResolution": true,
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "id": 22,
+ "panels": [],
+ "repeat": null,
+ "title": "Prometheus Metrics",
+ "type": "row"
+ },
+ {
+ "aliasColors": {
+ "Max": "#e24d42",
+ "Open": "#508642"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 25
+ },
+ "id": 18,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "process_max_fds{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Max",
+ "refId": "A"
+ },
+ {
+ "expr": "process_open_fds{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Open",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "File Descriptors",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Allocated bytes": "#7EB26D",
+ "Allocated bytes - 1m max": "#BF1B00",
+ "Allocated bytes - 1m min": "#BF1B00",
+ "Allocated bytes - 5m max": "#BF1B00",
+ "Allocated bytes - 5m min": "#BF1B00",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#447EBC"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "decimals": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 25
+ },
+ "id": 58,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/-/",
+ "fill": 0
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "process_resident_memory_bytes{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "RSS",
+ "metric": "process_resident_memory_bytes",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Target heap size",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "D",
+ "step": 10
+ },
+ {
+ "expr": "go_memstats_next_gc_bytes{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Next GC",
+ "metric": "go_memstats_next_gc_bytes",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "expr": "go_memstats_alloc_bytes{job=\"prometheus\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Allocated",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Memory",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Allocated bytes": "#F9BA8F",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#890F02"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 25
+ },
+ "id": 60,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\"}[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Allocated",
+ "metric": "go_memstats_alloc_bytes",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Allocations",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "Time series": "#70dbed"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 32
+ },
+ "id": 20,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_series",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Time series",
+ "metric": "prometheus_local_storage_memory_series",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Time series",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 32
+ },
+ "id": 24,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_active_appenders",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Head Appenders",
+ "metric": "prometheus_local_storage_memory_series",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Active Appenders",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "samples/s": "#e5a8e2"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 32
+ },
+ "id": 26,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_samples_appended_total[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Samples",
+ "metric": "prometheus_local_storage_ingested_samples_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Samples Appended",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "To persist": "#9AC48A"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 39
+ },
+ "id": 28,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/Max.*/",
+ "fill": 0
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_chunks",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Chunks",
+ "metric": "prometheus_local_storage_memory_chunks",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 39
+ },
+ "id": 30,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_chunks_created_total[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Created",
+ "metric": "prometheus_local_storage_chunk_ops_total",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks Created",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "Removed": "#e5ac0e"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 39
+ },
+ "id": 32,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_chunks_removed_total[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Removed",
+ "metric": "prometheus_local_storage_chunk_ops_total",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Chunks Removed",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max": "#447ebc",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "Min": "#447ebc",
+ "Now": "#7eb26d"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 46
+ },
+ "id": 34,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Max",
+ "fillBelowTo": "Min",
+ "lines": false
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_head_min_time",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Min",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "time() * 1000",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "Now",
+ "refId": "C"
+ },
+ {
+ "expr": "prometheus_tsdb_head_max_time",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Max",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head Time Range",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "dateTimeAsIso",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 46
+ },
+ "id": 36,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "GC Time",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Head GC Time/s",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 46
+ },
+ "id": 38,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Queue length",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_tsdb_blocks_loaded",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Blocks Loaded",
+ "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Blocks Loaded",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Failed Reloads": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 53
+ },
+ "id": 40,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_reloads_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Reloads",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TSDB Reloads",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ops",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 53
+ },
+ "id": 44,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_wal_corruptions_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "WAL Corruptions",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_tsdb_reloads_failures_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Reload Failures",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_tsdb_head_series_not_found[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Head Series Not Found",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_tsdb_compactions_failed_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Compaction Failures",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "D",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Retention Cutoff Failures",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "E",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "TSDB Problems",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 53
+ },
+ "id": 42,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Fsync Latency",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count[1m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Truncate Latency",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "WAL Latencies",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Failed Compactions": "#bf1b00",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 60
+ },
+ "id": 46,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compactions_total[10m])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Compactions",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Compactions",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 60
+ },
+ "id": 48,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Compaction Time",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Compaction Time",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Allocated bytes": "#F9BA8F",
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833",
+ "RSS": "#890F02"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 60
+ },
+ "id": 50,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_retention_cutoffs_total[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Retention Cutoffs",
+ "metric": "last",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Retention Cutoffs",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 67
+ },
+ "id": 56,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count[10m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Chunk Samples",
+ "metric": "prometheus_local_storage_series_chunks_persisted_count",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "First Compaction, Avg Chunk Samples",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "none",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 67
+ },
+ "id": 10,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_target_interval_length_seconds_count[5m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{ interval }}",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Target Scrapes",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 67
+ },
+ "id": 11,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{quantile}} ({{interval}})",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Scrape Duration",
+ "tooltip": {
+ "msResolution": true,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 74
+ },
+ "id": 62,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_http_request_duration_seconds_count[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{handler}}",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "HTTP requests",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "reqps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 74
+ },
+ "id": 64,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_http_request_duration_seconds_sum[1m]) / rate(prometheus_http_request_duration_seconds_count[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{handler}}",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "HTTP request latency",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 74
+ },
+ "id": 66,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_http_request_duration_seconds_sum[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{handler}}",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Time spent in HTTP requests",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "description": "Time spent in each mode, per second",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 81
+ },
+ "id": 68,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_engine_query_duration_seconds_sum[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{slice}}",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Query engine timings",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 81
+ },
+ "id": 70,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_rule_group_iterations_missed_total[1m]) ",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Rule group missed",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "expr": "rate(prometheus_rule_evaluation_failures_total[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Rule evals failed",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Rule group evaulation problems",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {
+ "Chunks": "#1F78C1",
+ "Chunks to persist": "#508642",
+ "Max chunks": "#052B51",
+ "Max to persist": "#3F6833"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 81
+ },
+ "id": 72,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(prometheus_rule_group_duration_seconds_sum[1m])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Rule evaluation duration",
+ "metric": "prometheus_local_storage_memory_chunkdescs",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Evaluation time of rule groups",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": "10s",
+ "schemaVersion": 12,
+ "version": 22,
+ "links": [],
+ "gnetId": null
+}
diff --git a/prometheus_grafana_loki/readme.md b/prometheus_grafana_loki/readme.md
new file mode 100644
index 0000000..73e256d
--- /dev/null
+++ b/prometheus_grafana_loki/readme.md
@@ -0,0 +1,1582 @@
+# Prometheus+Grafana in docker
+
+###### guide-by-example
+
+
+
+# Purpose
+
+Monitoring of a docker host and the cointaners.
+
+* [Official Prometheus](https://prometheus.io/)
+* [Official Grafana](https://grafana.com/)
+* [Official Loki](https://grafana.com/oss/loki/)
+
+Monitoring in this case means gathering and showing information on how services
+or machines or containers are running. Can be cpu, io, ram, disk use...
+can be number of http requests, errors, or results of backups.
+Prometheus deals with metrics. Loki deals with logs. Grafana is there to show
+the data on a dashboard.
+
+Lot of the prometheus stuff here is based off the magnificent
+[stefanprodan/dockprom.](https://github.com/stefanprodan/dockprom)
+
+# Chapters
+
+* **[Core prometheus+grafana](#Overview)** - nice dashboards with metrics of docker host and containers
+* **[PromQL](#PromQL)** - links to various learning resources
+* **[Pushgateway](#Pushgateway)** - push data to prometheus from anywhere
+* **[Alertmanager](#Alertmanager)** - setting alerts and getting notifications
+* **[Loki](#Loki)** - prometheus for logs
+* **[Minecraft Loki example](#minecraft-loki-example)** - logs, grafana alerts
+ and templates
+* **[Caddy reverse proxy monitoring](#caddy-reverse-proxy-monitoring)** -
+ metrics, logs and geoip map
+
+
+
+# Overview
+
+[Good youtube overview](https://youtu.be/h4Sl21AKiDg) of Prometheus.
+
+Prometheus is an open source system for monitoring and alerting,
+written in golang.
+It periodically collects **metrics** from configured **targets**,
+makes these metrics available for visualization, and can trigger **alerts**.
+Prometheus is relatively young project, it is a **pull type** monitoring.
+
+[Glossary.](https://prometheus.io/docs/introduction/glossary/)
+
+* **Prometheus Server** is the core of the system, responsible for
+ * pulling new metrics
+ * storing the metrics in a database and evaluating them
+ * making metrics available through PromQL API
+* **Targets** - machines, services, applications that are monitored.
+ These need to have an **exporter**.
+ * **exporter** - a script or a service that gathers metrics on the target,
+ converts them to prometheus server format,
+ and exposes them at an endpoint so they can be pulled
+* **Alertmanager** - responsible for handling alerts from Prometheus Server,
+ and sending notifications through email, slack, pushover,..
+ **In this setup [ntfy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal)
+ webhook will be used.**
+* **pushgateway** - allows push type of monitoring. Meaning a machine anywhere
+ in the world can push data in to your prometheus. Should not be overused
+ as it goes against the pull philosophy of prometheus.
+* **Grafana** - for web UI visualization of the collected metrics
+
+
+
+
+# Files and directory structure
+
+```
+/home/
+ └── ~/
+ └── docker/
+ └── prometheus/
+ ├── 🗁 grafana_data/
+ ├── 🗁 prometheus_data/
+ ├── 🗋 docker-compose.yml
+ ├── 🗋 .env
+ └── 🗋 prometheus.yml
+```
+
+* `grafana_data/` - a directory where grafana stores its data
+* `prometheus_data/` - a directory where prometheus stores its database and data
+* `.env` - a file containing environment variables for docker compose
+* `docker-compose.yml` - a docker compose file, telling docker how to run the containers
+* `prometheus.yml` - a configuration file for prometheus
+
+The three files must be provided.
+The directories are created by docker compose on the first run.
+
+# docker-compose
+
+* **Prometheus** - The official image used. Few extra commands passing configuration.
+ Of note is 240 hours(10days) **retention** policy.
+* **Grafana** - The official image used. Bind mounted directory
+ for persistent data storage. User sets **as root**, as it solves issues I am
+ lazy to investigate.
+* **NodeExporter** - An exporter for linux machines,
+ in this case gathering the **metrics** of the docker host,
+ like uptime, cpu load, memory use, network bandwidth use, disk space,...
+ Also **bind mount** of some system directories to have access to required info.
+* **cAdvisor** - An exporter for gathering docker **containers** metrics,
+ showing cpu, memory, network use of **each container**
+ Runs in `privileged` mode and has some bind mounts of system directories
+ to have access to required info.
+
+*Note* - ports are only `expose`, since expectation of use of a reverse proxy
+and accessing the services by hostname, not ip and port.
+
+`docker-compose.yml`
+```yml
+services:
+
+ # MONITORING SYSTEM AND THE METRICS DATABASE
+ prometheus:
+ image: prom/prometheus:v2.42.0
+ container_name: prometheus
+ hostname: prometheus
+ user: root
+ restart: unless-stopped
+ depends_on:
+ - cadvisor
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/etc/prometheus/console_libraries'
+ - '--web.console.templates=/etc/prometheus/consoles'
+ - '--storage.tsdb.retention.time=240h'
+ - '--web.enable-lifecycle'
+ volumes:
+ - ./prometheus_data:/prometheus
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ expose:
+ - "9090"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ # WEB BASED UI VISUALISATION OF METRICS
+ grafana:
+ image: grafana/grafana:9.4.3
+ container_name: grafana
+ hostname: grafana
+ user: root
+ restart: unless-stopped
+ env_file: .env
+ volumes:
+ - ./grafana_data:/var/lib/grafana
+ expose:
+ - "3000"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ # HOST LINUX MACHINE METRICS EXPORTER
+ nodeexporter:
+ image: prom/node-exporter:v1.5.0
+ container_name: nodeexporter
+ hostname: nodeexporter
+ restart: unless-stopped
+ command:
+ - '--path.procfs=/host/proc'
+ - '--path.rootfs=/rootfs'
+ - '--path.sysfs=/host/sys'
+ - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+ volumes:
+ - /proc:/host/proc:ro
+ - /sys:/host/sys:ro
+ - /:/rootfs:ro
+ expose:
+ - "9100"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ # DOCKER CONTAINERS METRICS EXPORTER
+ cadvisor:
+ image: gcr.io/cadvisor/cadvisor:v0.47.1
+ container_name: cadvisor
+ hostname: cadvisor
+ restart: unless-stopped
+ privileged: true
+ devices:
+ - /dev/kmsg:/dev/kmsg
+ volumes:
+ - /:/rootfs:ro
+ - /var/run:/var/run:ro
+ - /sys:/sys:ro
+ - /var/lib/docker:/var/lib/docker:ro
+ - /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
+ expose:
+ - "3000"
+ labels:
+ org.label-schema.group: "monitoring"
+
+networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+```
+
+`.env`
+
+```bash
+# GENERAL
+DOCKER_MY_NETWORK=caddy_net
+TZ=Europe/Bratislava
+
+# GRAFANA
+GF_SECURITY_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD=admin
+GF_USERS_ALLOW_SIGN_UP=false
+# GRAFANA EMAIL
+GF_SMTP_ENABLED=true
+GF_SMTP_HOST=smtp-relay.sendinblue.com:587
+GF_SMTP_USER=example@gmail.com
+GF_SMTP_PASSWORD=xzu0dfFhn3eqa
+```
+
+**All containers must be on the same network**.
+Which is named in the `.env` file.
+If one does not exist yet: `docker network create caddy_net`
+
+## prometheus.yml
+
+[Official documentation.](https://prometheus.io/docs/prometheus/latest/configuration/configuration/)
+
+Contains the bare minimum setup of targets from where metrics are to be pulled.
+
+`prometheus.yml`
+```yml
+global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'nodeexporter'
+ static_configs:
+ - targets: ['nodeexporter:9100']
+
+ - job_name: 'cadvisor'
+ static_configs:
+ - targets: ['cadvisor:8080']
+
+ - job_name: 'prometheus'
+ static_configs:
+ - targets: ['localhost:9090']
+```
+
+## Reverse proxy
+
+Caddy v2 is used, details
+[here](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy_v2).
+
+`Caddyfile`
+```php
+graf.{$MY_DOMAIN} {
+ reverse_proxy grafana:3000
+}
+
+prom.{$MY_DOMAIN} {
+ reverse_proxy prometheus:9090
+}
+```
+
+## First run and Grafana configuration
+
+* login admin/admin to `graf.example.com`, change the password
+* add Prometheus as a Data source in configuration
+ set URL to `http://prometheus:9090`
+* import dashboards from [json files in this repo](dashboards/)
+
+These **dashboards** are the preconfigured ones from
+[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom)
+with **few changes**.
+**Docker host** dashboard did not show free disk space for me, **had to change fstype**
+from `aufs` to `ext4`.
+Also included is [a fix](https://github.com/stefanprodan/dockprom/issues/18#issuecomment-487023049)
+for **host network monitoring** not showing traffick. In all of them
+the default time interval is set to 1h instead of 15m
+
+* **docker_host.json** - dashboard showing linux docker host metrics
+* **docker_containers.json** - dashboard showing docker containers metrics,
+ except the ones labeled as `monitoring` in the compose file
+* **monitoring_services.json** - dashboar showing docker containers metrics
+ of containers that are labeled `monitoring`
+
+
+
+---
+---
+
+# PromQL
+
+Some concept, highlights and examples.
+
+* [Stackoverflow - Prometheus instant vector vs range vector](https://stackoverflow.com/questions/68223824/prometheus-instant-vector-vs-range-vector)
+* [Short video](https://youtu.be/yLPTHinHB6Y)
+* [Prometheus Cheat Sheet - Basics \(Metrics, Labels, Time Series, Scraping\)](https://iximiuz.com/en/posts/prometheus-metrics-labels-time-series/)
+* [Learning Prometheus and PromQL - Learning Series](https://iximiuz.com/en/series/learning-prometheus-and-promql/)
+* [The official](https://prometheus.io/docs/prometheus/latest/querying/basics/)
+
+One thing to get from these is what kind of data a query in PromQL returns.
+Instant verctor vs range vector.
+
+---
+---
+
+# Pushgateway
+
+Gives freedom to **push** information in to prometheus from **anywhere**.
+
+### The setup
+
+To **add** pushgateway functionality to the current stack:
+
+* **New container** `pushgateway` added to the **compose** file.
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ # PUSHGATEWAY FOR PROMETHEUS
+ pushgateway:
+ image: prom/pushgateway:v1.5.1
+ container_name: pushgateway
+ hostname: pushgateway
+ restart: unless-stopped
+ command:
+ - '--web.enable-admin-api'
+ expose:
+ - "9091"
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+* Adding pushgateway to the **Caddyfile** of the reverse proxy so that
+ it can be reached at `https://push.example.com`
+
+
+ Caddyfile
+
+ ```php
+ push.{$MY_DOMAIN} {
+ reverse_proxy pushgateway:9091
+ }
+ ```
+
+
+* Adding pushgateway's **scrape point** to `prometheus.yml`
+
+
+ prometheus.yml
+
+ ```yml
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+ scrape_configs:
+ - job_name: 'pushgateway-scrape'
+ honor_labels: true
+ static_configs:
+ - targets: ['pushgateway:9091']
+ ```
+
+
+### The basics
+
+
+
+To **test pushing** some metric, execute in linux:
+`echo "some_metric 3.14" | curl --data-binary @- https://push.example.com/metrics/job/blabla/instance/whatever`
+
+You see **labels** being set to the pushed metric in the path.
+Label `job` is required, but after that it's whatever you want,
+though use of `instance` label is customary.
+Now in grafana, in **Explore** section you should see some results
+when quering for `some_metric`.
+
+The metrics sit on the pushgateway **forever**, unless deleted or container
+shuts down. **Prometheus will not remove** the metrics from it **after scraping**,
+it will keep scraping the pushgateway and store the value that sits there with
+the time of scraping.
+
+To **wipe** the pushgateway clean
+`curl -X PUT https://push.example.com/api/v1/admin/wipe`
+
+### The real world use
+
+[**Veeam Prometheus Grafana - guide-by-example**](https://github.com/DoTheEvo/veeam-prometheus-grafana)
+
+Linked above is much more on **pushgateway setup**,
+a real world use to **monitor backups**, along with **pushing metrics
+from windows** in powershell.
+
+
+
+---
+---
+
+# Alertmanager
+
+To send a **notification** about some **metric** breaching some preset **condition**.
+Notifications **chanels** set here will be **email** and
+[**ntfy**](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal)
+
+
+
+## The setup
+
+To **add** alertmanager to the current stack:
+
+* **New file** - `alertmanager.yml` will be **bind mounted** in alertmanager container.
+ This is the **configuration** on how and where **to deliver** alerts.
+
+
+ alertmanager.yml
+
+ ```yml
+ route:
+ receiver: 'email'
+
+ receivers:
+ - name: 'ntfy'
+ webhook_configs:
+ - url: 'https://ntfy.example.com/alertmanager'
+ send_resolved: true
+
+ - name: 'email'
+ email_configs:
+ - to: 'whoever@example.com'
+ from: 'alertmanager@example.com'
+ smarthost: smtp-relay.sendinblue.com:587
+ auth_username: ''
+ auth_identity: ''
+ auth_password: ''
+ ```
+
+
+* **New file** - `alert.rules` will be **bind mounted** in to prometheus container
+ This file **defines** at what value a metric becomes an **alert** event.
+
+
+ alert.rules
+
+ ```yml
+ groups:
+ - name: host
+ rules:
+ - alert: DiskSpaceLow
+ expr: sum(node_filesystem_free_bytes{fstype="ext4"}) > 19
+ for: 10s
+ labels:
+ severity: critical
+ annotations:
+ description: "Diskspace is low!"
+ ```
+
+
+* **Changed** `prometheus.yml`. Added **alerting section** that points to alertmanager
+ container, and also **set path** to a `rules` file.
+
+
+ prometheus.yml
+
+ ```yml
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+ scrape_configs:
+ - job_name: 'nodeexporter'
+ static_configs:
+ - targets: ['nodeexporter:9100']
+
+ - job_name: 'cadvisor'
+ static_configs:
+ - targets: ['cadvisor:8080']
+
+ - job_name: 'prometheus'
+ static_configs:
+ - targets: ['localhost:9090']
+
+ alerting:
+ alertmanagers:
+ - scheme: http
+ static_configs:
+ - targets:
+ - 'alertmanager:9093'
+
+ rule_files:
+ - '/etc/prometheus/rules/alert.rules'
+ ```
+
+
+* **New container** - `alertmanager` added to the compose file and **prometheus
+ container** has bind mount **rules file** added.
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ # MONITORING SYSTEM AND THE METRICS DATABASE
+ prometheus:
+ image: prom/prometheus:v2.42.0
+ container_name: prometheus
+ hostname: prometheus
+ restart: unless-stopped
+ user: root
+ depends_on:
+ - cadvisor
+ command:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--storage.tsdb.path=/prometheus'
+ - '--web.console.libraries=/etc/prometheus/console_libraries'
+ - '--web.console.templates=/etc/prometheus/consoles'
+ - '--storage.tsdb.retention.time=240h'
+ - '--web.enable-lifecycle'
+ volumes:
+ - ./prometheus_data:/prometheus
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ - ./alert.rules:/etc/prometheus/rules/alert.rules
+ expose:
+ - "9090"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ # ALERT MANAGMENT FOR PROMETHEUS
+ alertmanager:
+ image: prom/alertmanager:v0.25.0
+ container_name: alertmanager
+ hostname: alertmanager
+ restart: unless-stopped
+ volumes:
+ - ./alertmanager.yml:/etc/alertmanager.yml
+ - ./alertmanager_data:/alertmanager
+ command:
+ - '--config.file=/etc/alertmanager.yml'
+ - '--storage.path=/alertmanager'
+ expose:
+ - "9093"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+* **Adding** alertmanager to the **Caddyfile** of the reverse proxy so that
+ it can be reached at `https://alert.example.com`. **Not necessary**,
+ but useful as it **allows to send alerts from anywhere**,
+ not just from prometheus, or other containers on same docker network.
+
+
+ Caddyfile
+
+ ```php
+ alert.{$MY_DOMAIN} {
+ reverse_proxy alertmanager:9093
+ }
+ ```
+
+
+## The basics
+
+
+
+
+Once above setup is done **an alert** about low disk space **should fire**
+and a **notification** email should come.
+In `alertmanager.yml` switch from email **to ntfy** can be done.
+
+*Useful*
+
+* **alert** from anywhere using **curl**:
+ `curl -H 'Content-Type: application/json' -d '[{"labels":{"alertname":"blabla"}}]' https://alert.example.com/api/v1/alerts`
+* **reload rules**:
+ `curl -X POST https://prom.example.com/-/reload`
+
+[stefanprodan/dockprom](https://github.com/stefanprodan/dockprom#define-alerts)
+has more detailed section on alerting worth checking out.
+
+# Loki
+
+
+
+Loki is made by the grafana team. Sometimes called a Prometheus for logs,
+it's a **push** type monitoring, where an agent - **promtail**
+scrapes logs and then pushes them on to a Loki instance.
+
+For docker containers theres also an option to install
+[**loki-docker-driver**](https://grafana.com/docs/loki/latest/clients/docker-driver/)
+on a docker host and log pushing is set either globally in /etc/docker/daemon.json
+or per container in compose files.
+But as it turns out, **promtail** capabilities might be missed,
+like its ability to add labels to logs it scrapes based on some rule.
+Or processing data in some way, like translate IP addresses in to country
+names or cities.
+Still loki-docker-driver is useful for getting containers logs in to loki
+quickly and easily, with less cluttering of compose and less containers runnig.
+
+There will be **two examples** of logs monitoring.
+A **minecraft server** and a **caddy revers proxy**, both docker containers.
+
+## Loki setup
+
+* **New container** - `loki` added to the compose file.
+ Note the port 3100 is actually mapped to the host,
+ allowing `localhost:3100` from driver to work.
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ # LOG MANAGMENT WITH LOKI
+ loki:
+ image: grafana/loki:2.7.3
+ container_name: loki
+ hostname: loki
+ user: root
+ restart: unless-stopped
+ volumes:
+ - ./loki_data:/loki
+ - ./loki-config.yml:/etc/loki-config.yml
+ command:
+ - '-config.file=/etc/loki-config.yml'
+ ports:
+ - "3100:3100"
+ labels:
+ org.label-schema.group: "monitoring"
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+* **New file** - `loki-config.yml` bind mounted in the loki container.
+ The config comes from
+ [the official example](https://github.com/grafana/loki/tree/main/cmd/loki),
+ with some changes.
+ * **URL** changed for this setup.
+ * **Compactor** section is added, to have control over
+ [data retention.](https://grafana.com/docs/loki/latest/operations/storage/retention/)
+ * **Fixing** error - *"too many outstanding requests"*, discussion
+ [here.](https://github.com/grafana/loki/issues/5123)
+ It turns off parallelism, both split by time interval and shards split.
+
+
+ loki-config.yml
+
+ ```yml
+ auth_enabled: false
+
+ server:
+ http_listen_port: 3100
+
+ common:
+ path_prefix: /loki
+ storage:
+ filesystem:
+ chunks_directory: /loki/chunks
+ rules_directory: /loki/rules
+ replication_factor: 1
+ ring:
+ kvstore:
+ store: inmemory
+
+ # --- disable splitting to fix "too many outstanding requests"
+
+ query_range:
+ parallelise_shardable_queries: false
+
+ # --- compactor to have control over length of data retention
+
+ compactor:
+ working_directory: /loki/compactor
+ compaction_interval: 10m
+ retention_enabled: true
+ retention_delete_delay: 2h
+ retention_delete_worker_count: 150
+
+ limits_config:
+ retention_period: 240h
+ split_queries_by_interval: 0 # part of disable splitting fix
+
+ # -------------------------------------------------------
+
+ schema_config:
+ configs:
+ - from: 2020-10-24
+ store: boltdb-shipper
+ object_store: filesystem
+ schema: v11
+ index:
+ prefix: index_
+ period: 24h
+
+ ruler:
+ alertmanager_url: http://alertmanager:9093
+
+ analytics:
+ reporting_enabled: false
+ ```
+
+
+* #### loki-docker-driverdriver
+
+ * **Install** [loki-docker-driver](https://grafana.com/docs/loki/latest/clients/docker-driver/)
+ `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions`
+ To check if it's installed and enabled: `docker plugin ls`
+ * Containers that should be monitored usind loki-docker-driver need
+ `logging` section in their compose.
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ whoami:
+ image: "containous/whoami"
+ container_name: "whoami"
+ hostname: "whoami"
+ logging:
+ driver: "loki"
+ options:
+ loki-url: "http://localhost:3100/loki/api/v1/push"
+ ```
+
+
+* #### promtail
+
+ * Containers that should be monitored with **promtail** need it **added**
+ **to** their **compose** file, and made sure that it has access to the log files.
+
+
+ minecraft-docker-compose.yml
+
+ ```yml
+ services:
+
+ minecraft:
+ image: itzg/minecraft-server
+ container_name: minecraft
+ hostname: minecraft
+ restart: unless-stopped
+ env_file: .env
+ tty: true
+ stdin_open: true
+ ports:
+ - 25565:25565 # minecraft server players connect
+ volumes:
+ - ./minecraft_data:/data
+
+ # LOG AGENT PUSHING LOGS TO LOKI
+ promtail:
+ image: grafana/promtail
+ container_name: minecraft-promtail
+ hostname: minecraft-promtail
+ restart: unless-stopped
+ volumes:
+ - ./minecraft_data/logs:/var/log/minecraft:ro
+ - ./promtail-config.yml:/etc/promtail-config.yml
+ command:
+ - '-config.file=/etc/promtail-config.yml'
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+
+ caddy-docker-compose.yml
+
+ ```yml
+ services:
+
+ caddy:
+ image: caddy
+ container_name: caddy
+ hostname: caddy
+ restart: unless-stopped
+ env_file: .env
+ ports:
+ - "80:80"
+ - "443:443"
+ - "443:443/udp"
+ volumes:
+ - ./Caddyfile:/etc/caddy/Caddyfile
+ - ./caddy_config:/data
+ - ./caddy_data:/config
+ - ./caddy_logs:/var/log/caddy
+
+ # LOG AGENT PUSHING LOGS TO LOKI
+ promtail:
+ image: grafana/promtail
+ container_name: caddy-promtail
+ hostname: caddy-promtail
+ restart: unless-stopped
+ volumes:
+ - ./caddy_logs:/var/log/caddy:ro
+ - ./promtail-config.yml:/etc/promtail-config.yml
+ command:
+ - '-config.file=/etc/promtail-config.yml'
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+
+ ```
+
+
+ * Generic **config file for promtail**, needs to be bind mounted
+
+
+ promtail-config.yml
+
+ ```yml
+ clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+ scrape_configs:
+ - job_name: blablabla
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: blablabla_log
+ __path__: /var/log/blablabla/*.log
+ ```
+
+
+# Minecraft Loki example
+
+What can be seen in this example:
+
+* How to monitor logs of a docker container,
+ a [minecraft server](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/minecraft).
+* How to visualize the logs in a dashboard.
+* How to set an alert when a specific pattern appears in the logs.
+* How to extract information from log to include it in the alert notification.
+* Basic of grafana alert templates, so that the notification actually looks good,
+ and shows only relevant info.
+
+**Requirements** - grafana, loki, minecraft.
+
+
+
+### The Setup
+
+Initially **loki-docker-driver was used** to get logs to Loki, and it was simple
+and worked nicely. But during alert stage
+I **could not** figure out how to extract string from logs and include it in
+an **alert** notification. Specificly to not just say that "a player joined",
+but to have there name of the player that joined.
+The way to solve that was to **switch to promtail** and make use of its
+[pipeline_stages](https://grafana.com/docs/loki/latest/clients/promtail/pipelines/).
+Which was **suprisingly simple** and elegant.
+
+**Promtail** container is added to minecraft's **compose**, with bind mount
+access to minecraf's logs.
+
+
+minecraft-docker-compose.yml
+
+```yml
+services:
+
+ minecraft:
+ image: itzg/minecraft-server
+ container_name: minecraft
+ hostname: minecraft
+ restart: unless-stopped
+ env_file: .env
+ tty: true
+ stdin_open: true
+ ports:
+ - 25565:25565 # minecraft server players connect
+ volumes:
+ - ./minecraft_data:/data
+
+ # LOG AGENT PUSHING LOGS TO LOKI
+ promtail:
+ image: grafana/promtail
+ container_name: minecraft-promtail
+ hostname: minecraft-promtail
+ restart: unless-stopped
+ volumes:
+ - ./minecraft_data/logs:/var/log/minecraft:ro
+ - ./promtail-config.yml:/etc/promtail-config.yml
+ command:
+ - '-config.file=/etc/promtail-config.yml'
+
+networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+```
+
+
+**Promtail's config** is similar to the generic config in the previous section.
+The only addition is a short **pipeline** stage with a **regex** that runs against
+every log line before sending it to Loki. When a line matches, **a label** `player`
+is added to that log line.
+The value of that label comes from the **named capture group** thats part of
+that regex, the [syntax](https://www.regular-expressions.info/named.html)
+is: `(?Pgroup)`
+This label will be easy to use later in the alert stage.
+
+
+promtail-config.yml
+
+```yml
+clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+ - job_name: minecraft
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: minecraft_logs
+ __path__: /var/log/minecraft/*.log
+ pipeline_stages:
+ - regex:
+ expression: '.*:\s(?P.*)\sjoined the game$'
+ - labels:
+ player:
+```
+
+
+[Here's regex101](https://regex101.com/r/5vkOU2/1) of it,
+with some data to show how it works and bit of explanation.
+[Here's](https://stackoverflow.com/a/74962269/1383369)
+the stackoverflow answer that is the source for that config.
+
+
+
+### First steps in Grafana
+
+* In **grafana**, loki needs to be added as a **datasource**, `http://loki:3100`
+* In **Explore section**, filter, job = `minecraft_logs`, **Run query** button...
+ this should result in seeing minecraft logs and their volume/time graph.
+
+This Explore view will be recreated as a dashboard.
+
+### Dashboard minecraft_logs
+
+
+
+* New dashboard, new panel
+ * Data source - Loki
+ * Switch from `builder` to `code`
+ * query - `count_over_time({container_name="minecraft"} |= `` [1m])`
+ * Transform - Rename by regex - `(.*)` - `Logs`
+ * Graph type - `Time series`
+ * Title - Logs volume
+ * Transparent background
+ * Legend off
+ * Graph styles - bar
+ * Fill opacity - 50
+ * Color scheme - single color
+ * `Query options` - Min interval=1m
+ * Save
+* Add another pane to the dashboard
+ * Graph type - `Logs`
+ * Data source - Loki
+ * Switch from `builder` to `code`
+ query - `{container_name="minecraft"} |= ""`
+ * Title - *empty*
+ * Deduplication - Signature
+ * Save
+
+This should create a similar dashboard to the one in the picture above.
+
+[Performance tips](https://www.youtube.com/watch?v=YED8XIm0YPs)
+for grafana loki queries
+
+## Alerts in Grafana for Loki
+
+
+
+When **a player joins** minecraft server a **log appears** *"Bastard joined the game"*
+**Alert will be set** to look for string *"joined the game"* and **send notification**
+when it occurs.
+
+Now, might be good time to **brush up on PromQL / LogQL** and the **data types**
+they return when a query happens. That **instant vector** and **range vector**
+thingie. As grafana will scream when using range vector.
+
+### Create alert rule
+
+- **1 Set an alert rule name**
+ - Rule name = Minecraft-player-joined-alert
+- **2 Set a query and alert condition**
+ - **A** - Switch to Loki; set Last 5 minutes
+ - switch from builder to code
+ - `count_over_time({job="minecraft_logs"} |= "joined the game" [5m])`
+ - **B** - Reduce
+ - Function = Last
+ - Input = A
+ - Mode = Strict
+ - **C** - Treshold
+ - Input = B
+ - is above 0
+ - Make this the alert condition
+- **3 Alert evaluation behavior**
+ - Folder = "Alerts"
+ - Evaluation group (interval) = "five-min"
+ - Evaluation interval = 5m
+ - For 0s
+ - Configure no data and error handling
+ - Alert state if no data or all values are null = OK
+- **4 Add details for your alert rule**
+ - Here is where the label `player` that was set in **promtail** is used
+ Summary = `{{ $labels.player }} joined the Minecraft server.`
+ - Can also pass values from expressions by targeting A/B/C/.. from step2
+ Description = `Number of players that joined in the last 5 min: {{ $values.B }}`
+- **5 Notifications**
+ - nothing
+- Save and exit
+
+### Contact points
+
+ - New contact point
+ - Name = ntfy
+ - Integration = Webhook
+ - URL = https://ntfy.example.com/grafana
+ - Title = `{{ .CommonAnnotations.summary }}`
+ - Message = I put in [empty space unicode character](https://emptycharacter.com/)
+ - Disable resolved message = check
+ - Test
+ - Save
+
+### Notification policies
+
+ - Edit default
+ - Default contact point = ntfy
+ - Save
+
+After all this, there should be notification coming when a player joins.
+
+### grafana-to-ntfy
+
+For **alerts** one can use
+[ntfy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal)
+but on its own alerts from grafana are **just plain text json**.
+[Here's](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/gotify-ntfy-signal#grafana-to-ntfy)
+how to setup grafana-to-ntfy, to **make alerts look good**.
+
+
+
+---
+---
+
+### Templates
+
+Not really used here, but **they are pain** in the ass and there's some info
+as it took **embarrassingly** long to find that
+`{{ .CommonAnnotations.summary }}` for the title.
+
+* **Testing** should be done in **contact point** when editing,
+ useful **Test button** that allows you send alerts with custom values.
+* To [define a template.](https://i.imgur.com/ZczwCx2.png)
+* To [call a template.](https://i.imgur.com/0YdWA8Q.png)
+* My **big mistake** when playing with this was **missing a dot**.
+ In Contact point, in Title/Message input box.
+ * correct one - `{{ template "test" . }}`
+ * the one I had - `{{ template "test" }}`
+* So yeah, **dot** is important in here. It represents **data and context**
+ passed to a template. It can represent **global context**, or when used inside
+ `{{ range }}` it represents **iteration** loop value.
+* [This](https://pastebin.com/id3264k6) json structure is what an **alert** looks
+ like. Notice `alerts` being an **array** and `commonAnnotations` being **object**.
+ For **array** theres need to **loop** over it to get access to the
+ values in it. For **objects** one just needs to target **the name**
+ from global context.. **using dot** at the beginning.
+* To [iterate over alerts array.](https://i.imgur.com/gdwGhjN.png)
+* To just access a value - `{{ .CommonAnnotations.summary }}`
+* Then theres **conditional** things one can do in **golang templates**,
+ but I am not going to dig that deep...
+
+Templates resources
+
+* [Overview of Grafana Alerting and Message Templating for Slack](https://faun.pub/overview-of-grafana-alerting-and-message-templating-for-slack-6bb740ec44af)
+* [youtube - Unified Alerting Grafana 8 | Prometheus | Victoria | Telegraf | Notifications | Alert Templating](https://youtu.be/UtmmhLraSnE)
+* [Dot notation](https://www.practical-go-lessons.com/chap-32-templates#dot-notation)
+
+---
+---
+
+# Caddy reverse proxy monitoring
+
+What can be seen in this example:
+
+* Use of **Prometheus** to **monitor** a docker **container** -
+ [caddy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy).
+* How to **import a dashobard** to grafana.
+* Use of **Loki** to monitor **logs** of a docker **container**.
+* How to set **Promtail** to push only certain values and label them.
+* Create **dashboard** in grafana from data in **Loki**.
+
+**Requirements** - grafana, loki, caddy.
+
+
+
+**Reverse proxy** is kinda linchpin of a selfhosted setup as it is **in charge**
+of all the http/https **traffic** that goes in. So focus on monitoring this
+**keystone** makes sense.
+
+**Requirements** - grafana, prometheus, loki, caddy container
+
+## Caddy - Metrics - Prometheus
+
+
+
+**Caddy** has build in **exporter** of metrics for prometheus, so all that is needed
+is enabling it, **scrape it** by prometheus, and import a **dashboard**.
+
+* Edit Caddyfile to [enable metrics.](https://caddyserver.com/docs/metrics)
+
+
+ Caddyfile
+
+ ```php
+ {
+ servers {
+ metrics
+ }
+
+ admin 0.0.0.0:2019
+ }
+
+
+ a.{$MY_DOMAIN} {
+ reverse_proxy whoami:80
+ }
+ ```
+
+
+* Edit compose to publish 2019 port.
+ Likely **not necessary** if Caddy and Prometheus are on the **same docker network**,
+ but its nice to check if the metrics export works at `:2019/metrics`
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ caddy:
+ image: caddy
+ container_name: caddy
+ hostname: caddy
+ restart: unless-stopped
+ env_file: .env
+ ports:
+ - "80:80"
+ - "443:443"
+ - "443:443/udp"
+ - "2019:2019"
+ volumes:
+ - ./Caddyfile:/etc/caddy/Caddyfile
+ - ./caddy_config:/data
+ - ./caddy_data:/config
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+* Edit **prometheus.yml** to add caddy **scraping** point
+
+
+ prometheus.yml
+
+ ```yml
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+ scrape_configs:
+ - job_name: 'caddy'
+ static_configs:
+ - targets: ['caddy:2019']
+ ```
+
+
+* In grafana **import**
+ [caddy dashboard](https://grafana.com/grafana/dashboards/14280-caddy-exporter/)
+ or make your own, `caddy_reverse_proxy_upstreams_healthy` shows reverse proxy
+ upstreams, but thats all.
+
+But these **metrics** are more about **performance** and **load** put on Caddy,
+which in selfhosted environment will **likely be minimal** and not interesting.
+To get **more intriguing** info of who, when, **from where**, connects
+to what **service**,.. well for that monitoring of **access logs** is needed.
+
+---
+---
+
+## Caddy - Logs - Loki
+
+
+
+**Loki** itself just **stores** the logs. To get them to Loki a **Promtail** container is used
+that has **access** to caddy's **logs**. Its job is to **scrape** them regularly, maybe
+**process** them in some way, and then **push** them to Loki.
+Once there, a basic grafana **dashboard** can be made.
+
+
+### The setup
+
+* Have Grafana, Loki, Caddy working
+* Edit Caddy **compose**, bind mount `/var/log/caddy`.
+ **Add** to the compose also **Promtail container**, that has the same logs bind mount,
+ along with bind mount of its **config file**.
+ Promtail will scrape logs to which it now has access and **pushes** them **to Loki.**
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ caddy:
+ image: caddy
+ container_name: caddy
+ hostname: caddy
+ restart: unless-stopped
+ env_file: .env
+ ports:
+ - "80:80"
+ - "443:443"
+ - "443:443/udp"
+ - "2019:2019"
+ volumes:
+ - ./Caddyfile:/etc/caddy/Caddyfile
+ - ./caddy_data:/data
+ - ./caddy_config:/config
+ - ./caddy_logs:/var/log/caddy
+
+ # LOG AGENT PUSHING LOGS TO LOKI
+ promtail:
+ image: grafana/promtail
+ container_name: caddy-promtail
+ hostname: caddy-promtail
+ restart: unless-stopped
+ volumes:
+ - ./promtail-config.yml:/etc/promtail-config.yml
+ - ./caddy_logs:/var/log/caddy:ro
+ command:
+ - '-config.file=/etc/promtail-config.yml'
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+
+
+ promtail-config.yml
+
+ ```yml
+ clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+ scrape_configs:
+ - job_name: caddy_access_log
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: caddy_access_log
+ host: example.com
+ agent: caddy-promtail
+ __path__: /var/log/caddy/*.log
+ ```
+
+
+* **Promtail** scrapes a logs, **one line** at the time and is able to do **neat
+ things** with it before sending it - add labels, ignore entire line if some condition is met,
+ only send useful values,...
+ [Pipelines](https://grafana.com/docs/loki/latest/clients/promtail/pipelines/)
+ are used for this.
+ The **example here** is pretty straight forward, in **json** stage a desired
+ value is **extracted**, **template** is created with the use of that value,
+ and **output** is set.
+ [Here's](https://zerokspot.com/weblog/2023/01/25/testing-promtail-pipelines/)
+ some more to read with an example.
+
+
+ promtail-config.yml customizing fields
+
+ ```yml
+ clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+ scrape_configs:
+ - job_name: caddy_access_log
+
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: caddy_access_log
+ host: example.com
+ agent: caddy-promtail
+ __path__: /var/log/caddy/*.log
+
+ pipeline_stages:
+ - json:
+ expressions:
+ request_remote_ip: request.remote_ip
+
+ - template:
+ source: output # creates empty output variable
+ template: '{"remote_ip": {{.request_remote_ip}}}'
+
+ - output:
+ source: output
+
+ ```
+
+
+* Edit `Caddyfile` to enable
+ [**access logs**](https://caddyserver.com/docs/caddyfile/directives/log).
+ Unfortunetly this **can't be globally** enabled, so the easiest way seems to be
+ to create a **logging** [**snippet**](https://caddyserver.com/docs/caddyfile/concepts#snippets)
+ called `log_common` and copy paste the **import line** in to every site block.
+
+
+ Caddyfile
+
+ ```yml
+ (log_common) {
+ log {
+ output file /var/log/caddy/caddy_access.log
+ }
+ }
+
+ ntfy.example.com {
+ import log_common
+ reverse_proxy ntfy:80
+ }
+
+ mealie.{$MY_DOMAIN} {
+ import log_common
+ reverse_proxy mealie:80
+ }
+ ```
+
+
+* at this points logs should be visible and **explorable in grafana**
+ Explore > `{job="caddy_access_log"} |= "" | json`
+
+### Geoip
+
+
+
+**Promtail** got recently a **geoip stage**. One can feed it an **IP address** and an mmdb **geoIP
+datbase** and it adds geoip **labels** to the log entry.
+
+[The official documentation.](https://github.com/grafana/loki/blob/main/docs/sources/clients/promtail/stages/geoip.md)
+
+* **Register** a free account on [maxmind.com](https://www.maxmind.com/en/geolite2/signup).
+* **Download** one of the mmdb format **databases**
+ * `GeoLite2 City` - 70MB full geoip info - city, postal code, time zone, latitude/longitude,..
+ * `GeoLite2 Country` 6MB, just country and continent
+* **Bind mount** whichever database in to **promtail container**.
+
+
+ docker-compose.yml
+
+ ```yml
+ services:
+
+ caddy:
+ image: caddy
+ container_name: caddy
+ hostname: caddy
+ restart: unless-stopped
+ env_file: .env
+ ports:
+ - "80:80"
+ - "443:443"
+ - "443:443/udp"
+ - "2019:2019"
+ volumes:
+ - ./Caddyfile:/etc/caddy/Caddyfile
+ - ./caddy_data:/data
+ - ./caddy_config:/config
+ - ./caddy_logs:/var/log/caddy
+
+ # LOG AGENT PUSHING LOGS TO LOKI
+ promtail:
+ image: grafana/promtail
+ container_name: caddy-promtail
+ hostname: caddy-promtail
+ restart: unless-stopped
+ volumes:
+ - ./promtail-config.yml:/etc/promtail-config.yml
+ - ./caddy_logs:/var/log/caddy:ro
+ - ./GeoLite2-City.mmdb:/etc/GeoLite2-City.mmdb:ro
+ command:
+ - '-config.file=/etc/promtail-config.yml'
+
+ networks:
+ default:
+ name: $DOCKER_MY_NETWORK
+ external: true
+ ```
+
+* In **promtail** config, **json stage** is added where IP address is loaded in to
+ a **variable** called `remote_ip`, which then is used in **geoip stage**.
+ If all else is set correctly, the geoip **labels** are automaticly added to the log entry.
+
+
+ geoip promtail-config.yml
+
+ ```yml
+ clients:
+ - url: http://loki:3100/loki/api/v1/push
+
+ scrape_configs:
+ - job_name: caddy_access_log
+
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: caddy_access_log
+ host: example.com
+ agent: caddy-promtail
+ __path__: /var/log/caddy/*.log
+
+ pipeline_stages:
+ - json:
+ expressions:
+ remote_ip: request.remote_ip
+
+ - geoip:
+ db: "/etc/GeoLite2-City.mmdb"
+ source: remote_ip
+ db_type: "city"
+ ```
+
+
+Can be tested with opera build in VPN, or some online
+[site tester](https://pagespeed.web.dev/).
+
+### Dashboard
+
+
+
+* **new pane**, will be **time series** graph showing **Subdomains hits timeline**
+
+ * Graph type = Time series
+ * Data source = Loki
+ * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [1m])) by (request_host)`
+ * Query options > Min interval = 1m
+ * Transform > Rename by regex
+ * Match = `\{request_host="(.*)"\}`
+ * Replace = `$1`
+ * Title = "Subdomains hits timeline"
+ * Transparent
+ * Tooltip mode = All
+ * Tooltip values sort order = Descending
+ * Legen Placement = Right
+ * Value = Total
+ * Graph style = Bars
+ * Fill opacity = 50
+
+
+
+* Add **another pane**, will be a **pie chart**, showing **subdomains** divide
+
+ * Graph type = Pie chart
+ * Data source = Loki
+ * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [$__range])) by (request_host)`
+ * Query options > Min interval = 1m
+ * Transform > Rename by regex
+ * Match = `\{request_host="(.*)"\}`
+ * Replace = `$1`
+ * Title = "Subdomains use"
+ * Transparent
+ * Legen Placement = Right
+ * Value = Last
+
+
+
+* Add **another pane**, will be a **Geomap**, showing location of machine accessing
+ Caddy
+
+ * Graph type = Geomap
+ * Data source = Loki
+ * switch from builder to code
+ `{job="caddy_access_log"} |= "" | json`
+ * Query options > Min interval = 1m
+ * Transform > Extract fields
+ * Source = labels
+ * Format = JSON
+ * 1. Field = geoip_location_latitude; Alias = latitude
+ * 2. Field = geoip_location_longitude; Alias = longitude
+ * Title = "Geomap"
+ * Transparent
+ * Map view > View > *Drag and zoom around* > Use current map setting
+
+* Add **another pane**, will be a **pie chart**, showing **IPs** that hit the most
+
+ * Graph type = Pie chart
+ * Data source = Loki
+ * switch from builder to code
+ `sum(count_over_time({job="caddy_access_log"} |= "" | json [$__range])) by (request_remote_ip)`
+ * Query options > Min interval = 1m
+ * Transform > Rename by regex
+ * Match = `\{request_remote_ip="(.*)"\}`
+ * Replace = `$1`
+ * Title = "IPs by number of requests"
+ * Transparent
+ * Legen Placement = Right
+ * Value = Last or Total
+
+* Add **another pane**, this will be actual **log view**
+
+ * Graph type - Logs
+ * Data source - Loki
+ * Switch from builder to code
+ * query - `{job="caddy_access_log"} |= "" | json`
+ * Title - empty
+ * Deduplication - Exact or Signature
+ * Save
+
+
+
+
+# Update
+
+Manual image update:
+
+- `docker-compose pull`
+- `docker-compose up -d`
+- `docker image prune`
+
+# Backup and restore
+
+#### Backup
+
+Using [borg](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/borg_backup)
+that makes daily snapshot of the entire directory.
+
+#### Restore
+
+* down the prometheus containers `docker-compose down`
+* delete the entire prometheus directory
+* from the backup copy back the prometheus directory
+* start the containers `docker-compose up -d`