diff --git a/ansible/roles/grafana/files/dashboards/k8s-all-in-one.json b/ansible/roles/grafana/files/dashboards/k8s-all-in-one.json deleted file mode 100644 index d42ea5b..0000000 --- a/ansible/roles/grafana/files/dashboards/k8s-all-in-one.json +++ /dev/null @@ -1,2712 +0,0 @@ -{ - "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "7.0.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "This dashboard is one of major topic for Kubernetes Book \nand best optimized for virtual machine environment provided with \nhttps://github.com/sysnet4admin/_Book_k8sInfra\n\nNotice: It may be running on the other environment but cannot be guaranteed.", - "editable": true, - "gnetId": 13770, - "graphTooltip": 0, - "id": null, - "iteration": 1611497407892, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": "Prometheus", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 156, - "panels": [], - "title": "\ud074\ub7ec\uc2a4\ud130 \uc694\uc57d", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 9, - "x": 0, - "y": 1 - }, - "id": 158, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "avg(\navg_over_time((sum without ()(kube_pod_container_status_ready{namespace=\"kube-system\",pod=~\".*.dashboard.*|.*.dns.*|kube.*|.*.calico.*|.*.flannel.*|.*.etcd.*\"}) \n/ \ncount without ()(kube_pod_container_status_ready{namespace=\"kube-system\",pod=~\".*.dashboard.*|.*.dns.*|kube.*|.*.calico.*|.*.flannel.*|.*.etcd.*\"}))[$duration:5m])\n)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub9c8\uc2a4\ud130 \uc815\uc0c1 \uac00\ub3d9\ub960", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Docker version" - }, - "properties": [ - { - "id": "custom.width", - "value": 161 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Node name" - }, - "properties": [ - { - "id": "custom.width", - "value": 180 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "K8s version" - }, - "properties": [ - { - "id": "custom.width", - "value": 114 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "OS version" - }, - "properties": [ - { - "id": "custom.width", - "value": 249 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "pod_cidr" - }, - "properties": [ - { - "id": "custom.width", - "value": 113 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Kernel version" - }, - "properties": [ - { - "id": "custom.width", - "value": 297 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "role" - }, - "properties": [ - { - "id": "custom.width", - "value": 113 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Node status" - }, - "properties": [ - { - "id": "custom.width", - "value": 124 - }, - { - "id": "mappings", - "value": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "Good", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 1, - "operator": "", - "text": "Bad", - "to": "", - "type": 1, - "value": "0" - } - ] - }, - { - "id": "custom.displayMode", - "value": "color-background" - }, - { - "id": "custom.align", - "value": "center" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.width", - "value": 110 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "POD CIDR" - }, - "properties": [ - { - "id": "custom.width", - "value": 120 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod CIDR" - }, - "properties": [ - { - "id": "custom.width", - "value": 188 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 15, - "x": 9, - "y": 1 - }, - "id": 64, - "options": { - "angular": { - "decimals": 0, - "defaultColor": "#37872D", - "format": "none", - "linkIndex": "0", - "linkVars": [], - "mathColorValue": "data[end]", - "mathDisplayValue": "data[end]", - "mathScratchPad": "data = size(data)[1] == 0 ? [NaN] : data", - "options": {}, - "radius": "25px", - "thresholds": [ - { - "color": "#37872D", - "value": "1" - }, - { - "color": "#C4162A", - "value": "0" - } - ] - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Node name" - } - ] - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "kube_node_info", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{node}}", - "refId": "A" - }, - { - "expr": " (count by (node)(container_spec_memory_reservation_limit_bytes{pod!=\"\"}))", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "expr": "max\n by(node) (max by(instance) (kubelet_running_pod_count{job=\"kubelet\",metrics_path=\"/metrics\"})\n * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\",metrics_path=\"/metrics\"})\n", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "expr": "sum by (node)(label_replace(instance:node_cpu:ratio + on(instance) group_left(nodename) \nnode_uname_info,\"node\",\"$1\",\"nodename\", \"(.+)\") ) / sum by (node)(kube_node_status_capacity{resource=\"cpu\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "expr": "kube_node_role", - "format": "table", - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "expr": "label_replace(up{job=\"kubernetes-nodes\"}, \"node\", \"$1\", \"instance\", \"(.+)\")", - "format": "table", - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "expr": "label_replace(avg by (kubernetes_node) (time() - node_boot_time_seconds), \"node\", \"$1\", \"kubernetes_node\", \"(.+)\")", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub178\ub4dc \ud604\ud669", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "container_runtime_version", - "kernel_version", - "kubelet_version", - "node", - "os_image", - "pod_cidr", - "role", - "Value #G", - "Value #H" - ] - } - } - }, - { - "id": "seriesToColumns", - "options": { - "byField": "node" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "Value #G": 8, - "Value #H": 9, - "container_runtime_version": 7, - "instance": 2, - "kernel_version": 5, - "kubelet_version": 6, - "node": 0, - "os_image": 4, - "pod_cidr": 3, - "role": 1 - }, - "renameByName": { - "Value #C": "IP", - "Value #D": "Pod count", - "Value #E": "CPU usage", - "Value #G": "Node status", - "Value #H": "Uptime", - "container_runtime_version": "Docker version", - "instance": "IP", - "kernel_version": "Kernel version", - "kubelet_version": "K8s version", - "node": "Node name", - "os_image": "OS version", - "pod_cidr": "Pod CIDR", - "provider_id": "" - } - } - } - ], - "type": "table" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 4 - }, - "id": 131, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": " count(kube_namespace_created)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub124\uc784\uc2a4\ud398\uc774\uc2a4", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 4 - }, - "id": 159, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": " count(count by (pod)(container_spec_memory_reservation_limit_bytes{pod!=\"\"}))", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pods", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 4 - }, - "id": 160, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "count(kube_persistentvolumeclaim_info) ", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "PVC", - "type": "stat" - }, - { - "collapsed": false, - "datasource": "Prometheus", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 137, - "panels": [], - "repeat": null, - "title": "\uc804\uccb4 \ud074\ub7ec\uc2a4\ud130", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 4, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 35, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.5.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum by (instance,nodename) (irate(node_cpu_seconds_total{mode!~\"guest.*|idle|iowait\"}[$duration])) + on(instance) group_left(nodename) \nnode_uname_info) - 1", - "instant": false, - "interval": "", - "legendFormat": "{{nodename}} ", - "refId": "A" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.8, - "yaxis": "left" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\ub178\ub4dc\ubcc4 CPU \uc810\uc720\uc728", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 6, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 8 - }, - "hiddenSeries": false, - "id": 33, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "((node_memory_MemTotal_bytes + on(instance) group_left(nodename) \nnode_uname_info) - (node_memory_MemAvailable_bytes + on(instance) group_left(nodename) \nnode_uname_info))", - "instant": false, - "interval": "", - "legendFormat": "{{nodename}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\ub178\ub4dc\ubcc4 \uba54\ubaa8\ub9ac \uc0ac\uc6a9\ub7c9", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "", - "to": "", - "type": 1, - "value": "" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 8 - }, - "hiddenSeries": false, - "id": 133, - "legend": { - "alignAsTable": false, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "7.0.3", - "pointradius": 10, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:157", - "alias": "(in)", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate(node_network_receive_bytes_total[$duration]))", - "interval": "", - "legendFormat": "{{kubernetes_node}}(in)", - "refId": "A" - }, - { - "expr": "sum (rate(node_network_transmit_bytes_total[$duration]))", - "interval": "", - "legendFormat": "{{kubernetes_node}}(out)", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\ucd08\ub2f9 \ub124\ud2b8\uc6cc\ud06c \ud2b8\ub798\ud53d", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": "\uc218\uc2e0 | \uc1a1\uc2e0", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 8 - }, - "hiddenSeries": false, - "id": 164, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (verb) (rate(apiserver_request_total[$duration]))", - "interval": "", - "legendFormat": "{{verb}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "API \uc11c\ubc84 \ud638\ucd9c", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": "Prometheus", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 139, - "panels": [], - "repeat": null, - "title": "\ub178\ub4dc $node", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 0, - "y": 17 - }, - "id": 145, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum by (instance) (irate(node_cpu_seconds_total{mode!~\"guest.*|idle|iowait\", kubernetes_node=\"$node\"}[5m]))", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU \uc810\uc720\uc728", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 3, - "y": 17 - }, - "id": 146, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "(node_memory_MemTotal_bytes{kubernetes_node=\"m-k8s\"}-node_memory_MemAvailable_bytes{kubernetes_node=\"m-k8s\"})/node_memory_MemTotal_bytes{kubernetes_node=\"m-k8s\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\uba54\ubaa8\ub9ac \uc810\uc720\uc728", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 6, - "y": 17 - }, - "id": 154, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum(node_filesystem_size_bytes{kubernetes_node=\"$node\"} - node_filesystem_avail_bytes{kubernetes_node=\"$node\"}) by (kubernetes_node) / sum(node_filesystem_size_bytes{kubernetes_node=\"$node\"}) by (kubernetes_node)", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub514\uc2a4\ud06c \uc0ac\uc6a9\ub960(%)", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 9, - "y": 17 - }, - "id": 141, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "kube_node_status_allocatable_cpu_cores{node=\"$node\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU \ucf54\uc5b4", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 11, - "y": 17 - }, - "hiddenSeries": false, - "id": 162, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "switches", - "lines": false, - "pointradius": 1, - "points": true, - "yaxis": 2 - }, - { - "alias": "used filefd", - "color": "#F2495C" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node_filefd_allocated{instance=~\"$instance\"}", - "interval": "", - "intervalFactor": 5, - "legendFormat": "used filefd", - "refId": "A" - }, - { - "expr": "rate(node_context_switches_total{instance=~\"$instance\"}[$duration])", - "interval": "", - "intervalFactor": 5, - "legendFormat": "switches", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "File descriptor / Context switches", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "used filefd", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "context_switches", - "logBase": 2, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 17, - "y": 17 - }, - "hiddenSeries": false, - "id": 166, - "legend": { - "alignAsTable": false, - "avg": true, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*_Reads completed$/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_reads_completed_total{instance=~\"$instance\"}[$duration])", - "interval": "", - "legendFormat": "{{device}}_Reads", - "refId": "A" - }, - { - "expr": "rate(node_disk_writes_completed_total{instance=~\"$instance\"}[$duration])", - "interval": "", - "legendFormat": "{{device}}_Writes", - "refId": "B" - }, - { - "expr": "node_disk_io_now{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{device}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk IOPS \uc644\ub8cc", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "iops", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": 2 - } - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 9, - "y": 20 - }, - "id": 143, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "machine_memory_bytes{instance=\"$node\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\uba54\ubaa8\ub9ac", - "type": "stat" - }, - { - "collapsed": false, - "datasource": "Prometheus", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 168, - "panels": [], - "title": "Pod $pod", - "type": "row" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 0, - "y": 24 - }, - "id": 179, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "time()-kube_pod_start_time{pod=\"$pod\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pod \uc0dd\uc131\uc2dc\uac04", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "Bad", - "to": "", - "type": 1, - "value": "1" - }, - { - "from": "", - "id": 1, - "operator": "", - "text": "Good", - "to": "", - "type": 1, - "value": "0" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 5, - "y": 24 - }, - "id": 173, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum(kube_pod_container_status_waiting_reason{pod=\"$pod\", reason!~\"ContainerCreating\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pod \uac00\ub3d9 \uc0c1\ud0dc", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 10, - "y": 24 - }, - "id": 175, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum(round(increase(kube_pod_container_status_restarts_total{pod=\"$pod\"}[10m])))", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "10\ubd84 \uc774\ub0b4 \uc7ac\uc2dc\uc791", - "type": "stat" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPU(%)" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.9 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory(bytes)" - }, - "properties": [ - { - "id": "max", - "value": 1073741824 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "File system usage" - }, - "properties": [ - { - "id": "max", - "value": 1048576 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 13, - "y": 24 - }, - "id": 171, - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{pod=\"$pod\"})", - "interval": "", - "legendFormat": "Memory(bytes)", - "refId": "B" - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=\"$pod\"}[$duration]))", - "interval": "", - "legendFormat": "CPU(%)", - "refId": "A" - }, - { - "expr": "sum(container_fs_usage_bytes{pod=\"$pod\"})", - "interval": "", - "legendFormat": "File system usage", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub9ac\uc18c\uc2a4 \uc0ac\uc6a9\ub7c9", - "type": "bargauge" - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "noValue": "\u221e", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Memory" - }, - "properties": [ - { - "id": "unit", - "value": "decbytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU core" - }, - "properties": [ - { - "id": "unit", - "value": "core" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 24 - }, - "id": 177, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "pluginVersion": "7.0.3", - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{pod=\"$pod\"})", - "interval": "", - "legendFormat": "CPU core", - "refId": "A" - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{pod=\"$pod\"})", - "interval": "", - "legendFormat": "Memory", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "\ub9ac\uc18c\uc2a4 \ud560\ub2f9 \uc81c\ud55c", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "", - "to": "", - "type": 1, - "value": "" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 30 - }, - "hiddenSeries": false, - "id": 169, - "legend": { - "alignAsTable": false, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "7.0.3", - "pointradius": 10, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "in", - "color": "#73BF69", - "transform": "negative-Y" - }, - { - "alias": "out", - "color": "#5794F2" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_receive_bytes_total{pod=\"$pod\"}[$duration]))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "in", - "refId": "A" - }, - { - "expr": "sum(rate(container_network_transmit_bytes_total{pod=\"$pod\"}[$duration]))", - "interval": "", - "legendFormat": "out", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\ucd08\ub2f9 \ub124\ud2b8\uc6cc\ud06c \ud2b8\ub798\ud53d", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": "\uc218\uc2e0 | \uc1a1\uc2e0", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "", - "to": "", - "type": 1, - "value": "" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 30 - }, - "hiddenSeries": false, - "id": 180, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "7.0.3", - "pointradius": 10, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "in", - "color": "#73BF69", - "transform": "negative-Y" - }, - { - "alias": "out", - "color": "#5794F2" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_swap{pod=\"$pod\"}) by (pod)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Swap", - "refId": "A" - }, - { - "expr": "sum(rate(container_memory_failcnt{pod=\"$pod\"}[$duration])) by (pod)", - "interval": "", - "legendFormat": "Memory Failure", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\uba54\ubaa8\ub9ac \uc0c1\ud0dc(Swap/Failure)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 4, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 30 - }, - "hiddenSeries": false, - "id": 182, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "7.0.3", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:311", - "alias": "Thread", - "bars": true, - "color": "#FF9830", - "lines": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_threads{pod=\"$pod\", container!=\"POD\"})", - "interval": "", - "intervalFactor": 5, - "legendFormat": "Thread", - "refId": "A" - }, - { - "expr": "", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "\uc2a4\ub808\ub4dc \uac1c\uc218", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 25, - "style": "dark", - "tags": [ - "kubernetes", - "k8s" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "definition": "label_values(node_uname_info,kubernetes_node)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "node", - "options": [], - "query": "label_values(node_uname_info,kubernetes_node)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "5m", - "value": "5m" - }, - "hide": 0, - "label": null, - "name": "duration", - "options": [ - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": true, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "2m,5m,10m,30m,1h", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "definition": "query_result(node_uname_info{kubernetes_node=\"$node\"})", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": "query_result(node_uname_info{kubernetes_node=\"$node\"})", - "refresh": 1, - "regex": "/instance=\"(.+?)\"/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "definition": "label_values(kube_pod_labels, namespace)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "namespace", - "options": [], - "query": "label_values(kube_pod_labels, namespace)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "definition": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "pod", - "options": [], - "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "1 Kubernetes All-in-one Cluster Monitoring KR", - "uid": "4b545447f", - "version": 2 -} \ No newline at end of file diff --git a/ansible/roles/grafana/files/dashboards/k8s-cluster.json b/ansible/roles/grafana/files/dashboards/k8s-cluster.json new file mode 100644 index 0000000..8cf2dc0 --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/k8s-cluster.json @@ -0,0 +1,2635 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.6.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "$$hashKey": "object:3607", + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Summary metrics about containers running on Kubernetes nodes.\r\n\r\nDashboard was taken from here. This version does not reqiure you to\r\nsetup the Kubernetes-app plugin. (https://github.com/grafana/kubernetes-app)\r\n\r\nUse this Helm chart to launch Grafana into a Kubernetes cluster. It will include this dashboard and many more dashboards to give you visibility into the Kubernetes Cluster. (https://github.com/sekka1/cloud-public/tree/master/kubernetes/pods/grafana-helm)", + "editable": true, + "gnetId": 6417, + "graphTooltip": 1, + "id": null, + "iteration": 1528328092190, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": false, + "tags": [ + "kubernetes-app" + ], + "title": "Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Cluster Health", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_info{node=~\"$node\"}) / sum(kube_node_status_allocatable_pods{node=~\".*\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "80,90", + "title": "Cluster Pod Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{node=~\"$node\"}) / sum(kube_node_status_allocatable_cpu_cores{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "80,90", + "title": "Cluster CPU Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"}) / sum(kube_node_status_allocatable_memory_bytes{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "80,90", + "title": "Cluster Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum (node_filesystem_size{nodename=~\"$node\"}) - sum (node_filesystem_free{nodename=~\"$node\"})) / sum (node_filesystem_size{nodename=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "80,90", + "title": "Cluster Disk Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_pods{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_capacity_pods{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_info{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cluster Pod Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "pods", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity_cpu_cores{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cluster CPU Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocatable", + "refId": "A" + }, + { + "expr": "sum(kube_node_status_capacity_memory_bytes{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requested", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cluster Mem Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_filesystem_size{nodename=~\"$node\"}) - sum(node_filesystem_free{nodename=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "A" + }, + { + "expr": "sum(node_filesystem_size{nodename=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limit", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cluster Disk Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 14, + "panels": [], + "title": "Deployments", + "type": "row" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 16, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "row", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 0, + "pattern": "Metric", + "thresholds": [ + "0", + "0", + ".9" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "", + "colorMode": "row", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value", + "thresholds": [ + "0", + "1" + ], + "type": "number", + "unit": "none" + } + ], + "targets": [ + { + "expr": "kube_deployment_status_replicas{namespace=~\"$namespace\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ deployment }}", + "refId": "A" + } + ], + "title": "Deployment Replicas - Up To Date", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 11 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Deployment Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 11 + }, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_updated{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Deployment Replicas - Updated", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Deployment Replicas - Unavailable", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 22, + "panels": [], + "title": "Node", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_info{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Number Of Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 25, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_condition{condition=\"OutOfDisk\", node=~\"$node\", status=\"true\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1", + "title": "Nodes Out of Disk", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_spec_unschedulable{node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1", + "title": "Nodes Unavailable", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 28, + "panels": [], + "title": "Pods", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(78, 203, 42, 0.28)", + "full": false, + "lineColor": "#629e51", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Pods Running", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(78, 203, 42, 0.28)", + "full": false, + "lineColor": "#629e51", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Pods Pending", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(78, 203, 42, 0.28)", + "full": false, + "lineColor": "#629e51", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Pods Failed", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(78, 203, 42, 0.28)", + "full": false, + "lineColor": "#629e51", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Succeeded\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Pods Succeeded", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 34, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(78, 203, 42, 0.28)", + "full": false, + "lineColor": "#629e51", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Unknown\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Pods Unknown", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 36, + "panels": [], + "title": "Containers", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 38, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_status_running{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Containers Running", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 39, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_status_waiting{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Containers Waiting", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_status_terminated{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Containers Terminated", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(delta(kube_pod_container_status_restarts{namespace=\"kube-system\"}[30m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Containers Restarts (Last 30 Minutes)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 43, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=~\"$namespace\", node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU Cores Requested by Containers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=~\"$namespace\", node=~\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory Requested By Containers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 45, + "panels": [], + "title": "Jobs", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 47, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_job_status_succeeded{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Jobs Succeeded", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 48, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_job_status_active{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Jobs Succeeded", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 49, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_job_status_failed{namespace=~\"$namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Jobs Failed", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kubernetes", + "kubernetes-app" + ], + "templating": { + "list": [ + { + "current": { + "text": "No data sources found", + "value": "" + }, + "hide": 2, + "label": "", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/$ds/", + "type": "datasource" + }, + { + "current": { + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": null, + "name": "node", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "type": "constant" + }, + { + "current": { + "text": ".*", + "value": ".*" + }, + "hide": 0, + "label": null, + "name": "namespace", + "options": [ + { + "selected": true, + "text": ".*", + "value": ".*" + } + ], + "query": ".*", + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kubernetes Cluster (Prometheus)", + "uid": "4XuMd2Iiz", + "version": 1 +} \ No newline at end of file diff --git a/ansible/roles/grafana/files/dashboards/microwaveprop.json b/ansible/roles/grafana/files/dashboards/microwaveprop.json new file mode 100644 index 0000000..7020172 --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/microwaveprop.json @@ -0,0 +1,496 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": {"type": "grafana", "uid": "-- Grafana --"}, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "PromEx-exported metrics from microwaveprop (BEAM, Phoenix, Ecto, Microwaveprop.Instrument spans, Oban queue depth, ScoreCache/GridCache hit rates).", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "liveNow": false, + "refresh": "30s", + "schemaVersion": 39, + "tags": ["elixir", "phoenix", "promex", "microwaveprop"], + "templating": { + "list": [ + { + "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {"selected": true, "text": "All", "value": "$__all"}, + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "definition": "label_values(microwaveprop_prom_ex_beam_stats_process_count, instance)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "instance", + "options": [], + "query": {"query": "label_values(microwaveprop_prom_ex_beam_stats_process_count, instance)", "refId": "StandardVariableQuery"}, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Microwaveprop", + "uid": "microwaveprop", + "version": 1, + "weekStart": "", + "panels": [ + { + "type": "row", + "id": 100, + "title": "Overview", + "collapsed": false, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "panels": [] + }, + { + "type": "stat", + "id": 1, + "title": "Replicas Up", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"}, + "targets": [{"expr": "count(up{job=\"prop\"} == 1)", "refId": "A"}] + }, + { + "type": "stat", + "id": 2, + "title": "Uptime (Min Replica)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 60000}, {"color": "green", "value": 300000}]}, "unit": "ms"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "auto"}, + "targets": [{"expr": "min(microwaveprop_prom_ex_beam_stats_uptime_milliseconds_count{instance=~\"$instance\"})", "refId": "A"}] + }, + { + "type": "stat", + "id": 3, + "title": "Memory (Total)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "auto"}, + "targets": [{"expr": "sum(microwaveprop_prom_ex_beam_memory_allocated_bytes{instance=~\"$instance\"})", "refId": "A"}] + }, + { + "type": "stat", + "id": 4, + "title": "Process Count", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 5000}, {"color": "red", "value": 20000}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(microwaveprop_prom_ex_beam_stats_process_count{instance=~\"$instance\"})", "refId": "A"}] + }, + { + "type": "stat", + "id": 5, + "title": "Run Queue", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 20}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "max(microwaveprop_prom_ex_beam_stats_run_queue_count{instance=~\"$instance\"})", "refId": "A"}] + }, + { + "type": "stat", + "id": 6, + "title": "Reductions/s", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 1}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_reduction_count{instance=~\"$instance\"}[5m]))", "refId": "A"}] + }, + + {"type": "row", "id": 101, "title": "Phoenix HTTP", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "panels": []}, + { + "type": "timeseries", + "id": 10, + "title": "Request Rate (req/s)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "reqps"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_count{instance=~\"$instance\"}[1m]))", "legendFormat": "total", "refId": "A"} + ] + }, + { + "type": "timeseries", + "id": 11, + "title": "Endpoint Latency (p50 / p95 / p99)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 12, + "title": "Top Routes by p95 Latency", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "topk(8, histogram_quantile(0.95, sum by(le, route) (rate(microwaveprop_prom_ex_phoenix_router_dispatch_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m]))))", "legendFormat": "{{route}}", "refId": "A"} + ] + }, + { + "type": "timeseries", + "id": 13, + "title": "Top Routes by Request Rate", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "reqps"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "topk(8, sum by(route) (rate(microwaveprop_prom_ex_phoenix_router_dispatch_stop_duration_milliseconds_count{instance=~\"$instance\"}[5m])))", "legendFormat": "{{route}}", "refId": "A"} + ] + }, + + {"type": "row", "id": 102, "title": "BEAM Internals", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 22}, "panels": []}, + { + "type": "timeseries", + "id": 20, + "title": "Memory by Type", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 23}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 30, "lineWidth": 1, "stacking": {"mode": "normal"}}, "unit": "bytes"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(microwaveprop_prom_ex_beam_memory_processes_total_bytes{instance=~\"$instance\"})", "legendFormat": "processes", "refId": "A"}, + {"expr": "sum(microwaveprop_prom_ex_beam_memory_binary_total_bytes{instance=~\"$instance\"})", "legendFormat": "binary", "refId": "B"}, + {"expr": "sum(microwaveprop_prom_ex_beam_memory_ets_total_bytes{instance=~\"$instance\"})", "legendFormat": "ets", "refId": "C"}, + {"expr": "sum(microwaveprop_prom_ex_beam_memory_code_total_bytes{instance=~\"$instance\"})", "legendFormat": "code", "refId": "D"}, + {"expr": "sum(microwaveprop_prom_ex_beam_memory_atom_total_bytes{instance=~\"$instance\"})", "legendFormat": "atom", "refId": "E"}, + {"expr": "sum(microwaveprop_prom_ex_beam_memory_persistent_term_total_bytes{instance=~\"$instance\"})", "legendFormat": "persistent_term", "refId": "F"} + ] + }, + { + "type": "timeseries", + "id": 21, + "title": "Process / Port / ETS / Atom Counts", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 23}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(microwaveprop_prom_ex_beam_stats_process_count{instance=~\"$instance\"})", "legendFormat": "processes", "refId": "A"}, + {"expr": "sum(microwaveprop_prom_ex_beam_stats_port_count{instance=~\"$instance\"})", "legendFormat": "ports", "refId": "B"}, + {"expr": "sum(microwaveprop_prom_ex_beam_stats_ets_count{instance=~\"$instance\"})", "legendFormat": "ets tables", "refId": "C"}, + {"expr": "sum(microwaveprop_prom_ex_beam_stats_atom_count{instance=~\"$instance\"})", "legendFormat": "atoms", "refId": "D"} + ] + }, + { + "type": "timeseries", + "id": 22, + "title": "GC: Reclaimed Bytes/s + Count/s", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 31}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "Bps"}, "overrides": [{"matcher": {"id": "byName", "options": "gc count/s"}, "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}]}]}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_gc_reclaimed_bytes{instance=~\"$instance\"}[5m]))", "legendFormat": "reclaimed bytes/s", "refId": "A"}, + {"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_gc_count{instance=~\"$instance\"}[5m]))", "legendFormat": "gc count/s", "refId": "B"} + ] + }, + { + "type": "timeseries", + "id": 23, + "title": "Run Queue / Active Tasks / Context Switches", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 31}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max(microwaveprop_prom_ex_beam_stats_run_queue_count{instance=~\"$instance\"})", "legendFormat": "run queue", "refId": "A"}, + {"expr": "max(microwaveprop_prom_ex_beam_stats_active_task_count{instance=~\"$instance\"})", "legendFormat": "active tasks", "refId": "B"}, + {"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_context_switch_count{instance=~\"$instance\"}[5m]))", "legendFormat": "context switches/s", "refId": "C"} + ] + }, + + {"type": "row", "id": 103, "title": "Ecto / Postgres", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 39}, "panels": []}, + { + "type": "timeseries", + "id": 30, + "title": "Query Rate (queries/s)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ops"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum(rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_count{instance=~\"$instance\"}[5m]))", "legendFormat": "queries/s", "refId": "A"}] + }, + { + "type": "timeseries", + "id": 31, + "title": "Query Latency (p50 / p95 / p99)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 32, + "title": "Queue Time (waiting for connection) p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le, instance) (rate(microwaveprop_prom_ex_ecto_repo_query_queue_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95 — {{instance}}", "refId": "A"} + ] + }, + { + "type": "timeseries", + "id": 33, + "title": "Pool Size (configured)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(microwaveprop_prom_ex_ecto_repo_init_pool_size{instance=~\"$instance\"})", "legendFormat": "pool size", "refId": "A"}] + }, + + {"type": "row", "id": 104, "title": "Upstream HTTP Clients (Microwaveprop.Instrument)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 56}, "panels": []}, + { + "type": "timeseries", + "id": 40, + "title": "Weather/Forecast Fetch p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 57}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_grid_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_grid", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_profile", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_idx_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_idx", "refId": "C"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_download_grib_ranges_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.download_grib_ranges", "refId": "D"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_gefs_fetch_grid_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "gefs.fetch_grid_profiles", "refId": "E"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_narr_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "narr.fetch_profile", "refId": "F"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_rtma_fetch_observation_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "rtma.fetch_observation", "refId": "G"} + ] + }, + { + "type": "timeseries", + "id": 41, + "title": "Surface/Sounding/Radar Fetch p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 57}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_asos_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_asos", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_raob_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_raob", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_iemre_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_iemre", "refId": "C"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_uwyo_fetch_sounding_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "uwyo.fetch_sounding", "refId": "D"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_ncei_fetch_metar_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "ncei.fetch_metar", "refId": "E"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_nexrad_fetch_frame_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "nexrad.fetch_frame", "refId": "F"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_nexrad_decode_png_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "nexrad.decode_png", "refId": "G"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_mrms_list_latest_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "mrms.list_latest", "refId": "H"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_mrms_download_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "mrms.download", "refId": "I"} + ] + }, + { + "type": "timeseries", + "id": 42, + "title": "Ancillary Service Fetch p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 65}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_solar_fetch_indices_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "solar.fetch_indices", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_swpc_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "swpc.fetch", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_giro_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "giro.fetch", "refId": "C"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_qrz_request_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "qrz.request", "refId": "D"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_geocoder_geocode_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "geocoder.geocode", "refId": "E"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_elevation_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "elevation.fetch_profile", "refId": "F"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_srtm_download_tile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "srtm.download_tile", "refId": "G"} + ] + }, + { + "type": "timeseries", + "id": 43, + "title": "Upstream Call Rate (calls/s)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 65}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ops"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "topk(10, sum by(__name__) (rate({__name__=~\"microwaveprop_(hrrr|iem|nexrad|gefs|narr|uwyo|elevation|mrms|rtma|ncei|solar|swpc|giro|qrz|geocoder|srtm)_.+_count\", instance=~\"$instance\"}[5m])))", "legendFormat": "{{__name__}}", "refId": "A"} + ] + }, + + {"type": "row", "id": 105, "title": "Workers / Pipeline Phases", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 73}, "panels": []}, + { + "type": "timeseries", + "id": 50, + "title": "Worker Phase Duration p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 74}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_grid_worker_perform_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "propagation.grid_worker.perform", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_grid_score_band_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "propagation_grid.score_band", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_terrain_analyse_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "terrain.analyse", "refId": "C"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_radar_aggregate_stats_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "radar.aggregate_stats", "refId": "D"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_terrain_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.terrain_profile", "refId": "E"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_mechanism_classify_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.mechanism_classify", "refId": "F"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_mrms_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.mrms_fetch", "refId": "G"} + ] + }, + { + "type": "timeseries", + "id": 51, + "title": "PropagationGridWorker — Tasks Seeded + Exceptions", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 74}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "bars", "fillOpacity": 50}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max(microwaveprop_propagation_grid_worker_seeded_queued_tasks{instance=~\"$instance\"})", "legendFormat": "queued tasks (last seed)", "refId": "A"}, + {"expr": "sum(rate(microwaveprop_propagation_grid_worker_exception_count{instance=~\"$instance\"}[5m]))", "legendFormat": "exceptions/s", "refId": "B"} + ] + }, + + {"type": "row", "id": 106, "title": "LiveView Hot Path", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 82}, "panels": []}, + { + "type": "timeseries", + "id": 60, + "title": "Map LiveView Calls p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 83}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_scores_at_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "scores_at", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_point_forecast_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "point_forecast", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_point_detail_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "point_detail", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 61, + "title": "Cache Hit Ratio (last 5m)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 83}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(rate(microwaveprop_propagation_scores_at_cache_count{hit=\"true\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(microwaveprop_propagation_scores_at_cache_count{instance=~\"$instance\"}[5m])), 1e-9)", "legendFormat": "ScoreCache", "refId": "A"}, + {"expr": "sum(rate(microwaveprop_weather_grid_cache_lookup_count{hit=\"true\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(microwaveprop_weather_grid_cache_lookup_count{instance=~\"$instance\"}[5m])), 1e-9)", "legendFormat": "Weather GridCache", "refId": "B"} + ] + }, + + {"type": "row", "id": 107, "title": "DB Batches + Subprocesses", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 91}, "panels": []}, + { + "type": "timeseries", + "id": 70, + "title": "DB Upsert Batch Duration p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 92}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_upsert_hrrr_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "upsert_hrrr_profiles", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_upsert_gefs_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "upsert_gefs_profiles", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_replace_scores_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "replace_scores", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 71, + "title": "wgrib2 Subprocess Duration p95", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 92}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_from_file_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid_from_file", "refId": "B"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_from_file_mapped_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid_from_file_mapped", "refId": "C"} + ] + }, + + {"type": "row", "id": 108, "title": "Oban Queue Depth (sampled every 30s)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 100}, "panels": []}, + { + "type": "timeseries", + "id": 80, + "title": "Available Jobs by Queue", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 101}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "stacking": {"mode": "normal"}}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max by(queue) (microwaveprop_oban_queue_count{state=\"available\", instance=~\"$instance\"})", "legendFormat": "{{queue}}", "refId": "A"} + ] + }, + { + "type": "timeseries", + "id": 81, + "title": "Executing Jobs by Queue", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 101}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "stacking": {"mode": "normal"}}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max by(queue) (microwaveprop_oban_queue_count{state=\"executing\", instance=~\"$instance\"})", "legendFormat": "{{queue}}", "refId": "A"} + ] + }, + { + "type": "timeseries", + "id": 82, + "title": "Retryable + Discarded Jobs by Queue", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 109}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max by(queue, state) (microwaveprop_oban_queue_count{state=~\"retryable|discarded|cancelled\", instance=~\"$instance\"})", "legendFormat": "{{queue}} / {{state}}", "refId": "A"} + ] + } + ] +}