From 1ddacbb8093b67c1a612a8cf8cf740e41028bb87 Mon Sep 17 00:00:00 2001 From: Beast Date: Tue, 4 Aug 2026 18:51:23 +0800 Subject: [PATCH] feat: add quersi monitoring --- README.md | 1 + .../infrastructure/quersi-host.json | 674 ++++++++++++++++++ grafana/provisioning/alerting/rules.yml | 78 ++ prometheus/prometheus.yml | 26 + 4 files changed, 779 insertions(+) create mode 100644 grafana/dashboards/infrastructure/quersi-host.json diff --git a/README.md b/README.md index 57f7ec4..2612fd1 100644 --- a/README.md +++ b/README.md @@ -560,6 +560,7 @@ All chain dashboards share a chain selector and link to each other via the **Cha | **Support Host** | Support server system metrics | | **Senoti Host** | Senoti fleet system metrics | | **Subsquid Host** | Subsquid fleet system metrics | +| **Quersi Host** | Quersi wallet remote-config system metrics | ### Applications diff --git a/grafana/dashboards/infrastructure/quersi-host.json b/grafana/dashboards/infrastructure/quersi-host.json new file mode 100644 index 0000000..0d6f087 --- /dev/null +++ b/grafana/dashboards/infrastructure/quersi-host.json @@ -0,0 +1,674 @@ +{ + "title": "Quersi Host", + "uid": "infra-quersi-host", + "tags": ["infrastructure", "quersi"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "10s", + "editable": true, + "panels": [ + { + "type": "text", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "mode": "markdown", + "content": "# Quersi Host System Metrics\n\nCapacity and health for the Quersi wallet remote-config host (`qrc-1.quantus.com`)." + } + }, + { + "type": "stat", + "title": "Host Status", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 10, + "targets": [ + { + "expr": "up{job=\"quersi-1\"}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ] + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 1, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + } + } + }, + { + "type": "stat", + "title": "CPU Usage", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 2, + "targets": [ + { + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", job=\"quersi-1\"}[5m])) * 100)", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + }, + "text": { + "titleSize": 16, + "valueSize": 48 + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 60, + "color": "yellow" + }, + { + "value": 80, + "color": "red" + } + ] + } + } + } + }, + { + "type": "stat", + "title": "Memory Usage", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 3, + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes{job=\"quersi-1\"} - node_memory_MemAvailable_bytes{job=\"quersi-1\"}) / node_memory_MemTotal_bytes{job=\"quersi-1\"}) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + }, + "text": { + "titleSize": 16, + "valueSize": 48 + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 90, + "color": "red" + } + ] + } + } + } + }, + { + "type": "stat", + "title": "Disk Usage", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 4, + "targets": [ + { + "expr": "((node_filesystem_size_bytes{job=\"quersi-1\", mountpoint=\"/\"} - node_filesystem_avail_bytes{job=\"quersi-1\", mountpoint=\"/\"}) / node_filesystem_size_bytes{job=\"quersi-1\", mountpoint=\"/\"}) * 100", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + }, + "text": { + "titleSize": 16, + "valueSize": 48 + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 75, + "color": "yellow" + }, + { + "value": 85, + "color": "red" + } + ] + } + } + } + }, + { + "type": "timeseries", + "title": "CPU Usage Over Time", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 5, + "targets": [ + { + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\", job=\"quersi-1\"}[5m])) * 100)", + "refId": "A", + "legendFormat": "CPU Usage", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "calcs": ["lastNotNull", "mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + } + } + }, + { + "type": "timeseries", + "title": "Memory Usage Over Time", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 6, + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes{job=\"quersi-1\"} - node_memory_MemAvailable_bytes{job=\"quersi-1\"}) / node_memory_MemTotal_bytes{job=\"quersi-1\"}) * 100", + "refId": "A", + "legendFormat": "Memory Usage", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "calcs": ["lastNotNull", "mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + } + } + }, + { + "type": "timeseries", + "title": "Network Traffic", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 7, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"quersi-1\", device!~\"lo|docker.*|veth.*\"}[5m])", + "refId": "A", + "legendFormat": "{{device}} - RX", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "rate(node_network_transmit_bytes_total{job=\"quersi-1\", device!~\"lo|docker.*|veth.*\"}[5m])", + "refId": "B", + "legendFormat": "{{device}} - TX", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "calcs": ["lastNotNull", "mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + } + } + }, + { + "type": "timeseries", + "title": "Load Average", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 8, + "targets": [ + { + "expr": "node_load1{job=\"quersi-1\"}", + "refId": "A", + "legendFormat": "1m", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "node_load5{job=\"quersi-1\"}", + "refId": "B", + "legendFormat": "5m", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "node_load15{job=\"quersi-1\"}", + "refId": "C", + "legendFormat": "15m", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 2, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + } + } + }, + { + "type": "stat", + "title": "System Uptime", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 9, + "targets": [ + { + "expr": "node_time_seconds{job=\"quersi-1\"} - node_boot_time_seconds{job=\"quersi-1\"}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + }, + "text": { + "titleSize": 16, + "valueSize": 32 + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + } + } + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "7d", "30d"] + } +} diff --git a/grafana/provisioning/alerting/rules.yml b/grafana/provisioning/alerting/rules.yml index d21b5a6..eab3f40 100644 --- a/grafana/provisioning/alerting/rules.yml +++ b/grafana/provisioning/alerting/rules.yml @@ -1148,6 +1148,84 @@ groups: chain: planck severity: critical + # =============================================== + # QUERSI HOST ALERTS (wallet remote config — no chain) + # =============================================== + - uid: quersi_host_down + title: Quersi Host Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: up{job="quersi-1"} + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + expression: B + refId: C + type: threshold + noDataState: Alerting + execErrState: Alerting + for: 3m + annotations: + description: 'Quersi host ({{ index $labels "instance" }}) is down for more than 3 minutes — wallet remote config unavailable' + summary: 'Quersi Host is DOWN' + labels: + severity: critical + # =============================================== # TASK MASTER MONITORING ALERTS (Planck - High Priority) # =============================================== diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index b39ccc8..ba3e769 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -502,6 +502,32 @@ scrape_configs: - senoti-core-1.quantus.com labels: chain: planck +- job_name: quersi-1 + honor_timestamps: true + track_timestamps_staleness: false + scrape_interval: 15s + scrape_timeout: 10s + scrape_protocols: + - PrometheusText0.0.4 + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText1.0.0 + metrics_path: /metrics + scheme: https + enable_compression: true + follow_redirects: true + enable_http2: true + fallback_scrape_protocol: PrometheusText0.0.4 + http_headers: + CF-Access-Client-Id: + files: + - /etc/prometheus/secrets/cf_access_client_id + CF-Access-Client-Secret: + files: + - /etc/prometheus/secrets/cf_access_client_secret + static_configs: + - targets: + - qrc-1.quantus.com - job_name: planck-chain honor_timestamps: true track_timestamps_staleness: false