DolphinScheduler 挂了没人知道？Prometheus + Grafana 监控兜底方案，亲测可用

curl http://<dolphin-host>:12345/dolphinscheduler/actuator/prometheus

scrape_configs:  # 主机监控（可选，用于监控 Dolphin 所在服务器资源）  - job_name: 'node-exporter'    static_configs:      - targets:          - '<dolphin-host>:9100'        labels:          instance: 'dolphin-server'          nodename: 'dolphin-server'  # DolphinScheduler 指标采集  - job_name: 'dolphinscheduler'    static_configs:      - targets: ['<dolphin-host>:12345']        labels:          service: dolphinscheduler    metrics_path: '/dolphinscheduler/actuator/prometheus'    metric_relabel_configs:      - source_labels: [application]        target_label: app      - regex: 'application'        action: labeldrop

# DolphinScheduler 监控告警规则groups:  - name: dolphinscheduler-alerts    rules:      # ============ 核心规则：服务存活检测 ============      # 规则1：一小时内没有成功的任务（大概率已经挂了）      - alert: DolphinScheduler一小时无成功任务        expr: increase(ds_task_instance_count_total{state="success"}[1h]) == 0        for: 5m        labels:          severity: critical          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 一小时内无成功任务"          description: "实例 {{ $labels.application }} 在过去 1 小时内没有产生成功的任务，可能已经挂掉，请立即检查服务状态"      # 规则2：30分钟内没有成功的任务（预警）      - alert: DolphinScheduler30分钟无成功任务        expr: increase(ds_task_instance_count_total{state="success"}[30m]) == 0        for: 5m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 30分钟内无成功任务"          description: "实例 {{ $labels.application }} 在过去 30 分钟内没有产生成功的任务，请注意检查"      # ============ 任务健康度检测 ============      # 规则3：任务失败率过高      - alert: DolphinScheduler任务失败率高        expr: |          (            increase(ds_task_instance_count_total{state="fail"}[10m])            /            (increase(ds_task_instance_count_total{state="finish"}[10m]) + 1)          ) > 0.1        for: 5m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 任务失败率过高"          description: "实例 {{ $labels.application }} 在过去 10 分钟内任务失败率超过 10%，当前失败率: {{ $value | humanizePercentage }}"      # 规则4：任务分发失败      - alert: DolphinScheduler任务分发失败        expr: increase(ds_task_dispatch_failure_count_total[10m]) > 0        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 任务分发失败"          description: "实例 {{ $labels.application }} 在过去 10 分钟内有 {{ $value }} 次任务分发失败"      # ============ Worker 健康检测 ============      # 规则5：Worker 活跃线程数为 0（有任务提交却没人干活）      - alert: DolphinScheduler Worker无活跃线程        expr: ds_worker_active_execute_thread == 0 and increase(ds_task_instance_count_total{state="submit"}[5m]) > 0        for: 10m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler Worker 无活跃执行线程"          description: "实例 {{ $labels.application }} 的 Worker 没有活跃的执行线程，但有任务提交，任务可能无法执行"      # 规则6：Worker 内存使用率过高      - alert: DolphinScheduler Worker内存使用率高        expr: ds_worker_memory_usage > 0.85        for: 5m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler Worker 内存使用率过高"          description: "实例 {{ $labels.application }} 的 Worker 内存使用率为 {{ $value | humanizePercentage }}，超过 85%"      # 规则7：Worker CPU 使用率过高      - alert: DolphinScheduler Worker CPU使用率高        expr: ds_worker_cpu_usage > 0.85        for: 5m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler Worker CPU 使用率过高"          description: "实例 {{ $labels.application }} 的 Worker CPU 使用率为 {{ $value | humanizePercentage }}，超过 85%"      # ============ 工作流 & Master 检测 ============      # 规则8：工作流实例长时间运行（可能阻塞了）      - alert: DolphinScheduler工作流长时间运行        expr: ds_workflow_instance_running > 10        for: 30m        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 工作流长时间运行"          description: "实例 {{ $labels.application }} 有 {{ $value }} 个工作流实例运行超过 30 分钟，可能存在阻塞"      # 规则9：Master 故障转移检查异常      - alert: DolphinScheduler Master故障转移检查异常        expr: increase(ds_master_scheduler_failover_check_count_total{result="fail"}[10m]) > 0        labels:          severity: critical          group: dolphinscheduler        annotations:          summary: "DolphinScheduler Master 故障转移检查异常"          description: "实例 {{ $labels.application }} 在过去 10 分钟内 Master 故障转移检查失败 {{ $value }} 次"      # ============ 告警通道自检 ============      # 规则10：告警发送失败（监控系统的监控，套娃了属于是）      - alert: DolphinScheduler告警发送失败        expr: increase(ds_alert_send_count_total{status="fail"}[10m]) > 5        labels:          severity: warning          group: dolphinscheduler        annotations:          summary: "DolphinScheduler 告警发送失败"          description: "实例 {{ $labels.application }} 在过去 10 分钟内有 {{ $value }} 次告警发送失败"

{"__inputs":[{"name":"DS_PROMETHEUS","label":"Prometheus","description":"","type":"datasource","pluginId":"prometheus","pluginName":"Prometheus"}],"__requires":[{"type":"grafana","id":"grafana","name":"Grafana","version":"9.0.0"},{"type":"datasource","id":"prometheus","name":"Prometheus","version":"1.0.0"},{"type":"panel","id":"stat","name":"Stat","version":""},{"type":"panel","id":"gauge","name":"Gauge","version":""},{"type":"panel","id":"timeseries","name":"Time series","version":""},{"type":"panel","id":"piechart","name":"Pie chart","version":""},{"type":"panel","id":"bargauge","name":"Bar gauge","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":null,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":4,"x":0,"y":0},"id":1,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"textMode":"auto"},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_task_instance_count_total{state=\"success\"}","refId":"A"}],"title":"任务成功总数","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":1}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":4,"x":4,"y":0},"id":2,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"textMode":"auto"},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_task_instance_count_total{state=\"fail\"}","refId":"A"}],"title":"任务失败总数","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":4,"x":8,"y":0},"id":3,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"textMode":"auto"},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_workflow_instance_running","refId":"A"}],"title":"运行中工作流数","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0.7},{"color":"red","value":0.85}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":12,"y":0},"id":4,"options":{"orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_worker_memory_usage","refId":"A"}],"title":"Worker 内存使用率","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0.7},{"color":"red","value":0.85}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":0},"id":5,"options":{"orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_worker_cpu_usage","refId":"A"}],"title":"Worker CPU 使用率","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"success"},"properties":[{"id":"color","value":{"fixedColor":"green","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"fail"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":4},"id":6,"options":{"legend":{"calcs":["last","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_task_instance_count_total{state=\"success\"}[5m])","legendFormat":"success","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_task_instance_count_total{state=\"fail\"}[5m])","legendFormat":"fail","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_task_instance_count_total{state=\"timeout\"}[5m])","legendFormat":"timeout","refId":"C"}],"title":"任务执行趋势（5分钟增量）","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"tooltip":false,"viz":false,"legend":false}},"mappings":[],"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":4},"id":7,"options":{"legend":{"displayMode":"table","placement":"right","showLegend":true,"values":["value","percent"]},"pieType":"pie","tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_task_instance_count_total","legendFormat":"{{state}}","refId":"A"}],"title":"任务状态分布","type":"piechart"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0.05},{"color":"red","value":0.1}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":18,"y":4},"id":8,"options":{"orientation":"auto","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(ds_task_instance_count_total{state=\"fail\"}[10m]) / (rate(ds_task_instance_count_total{state=\"finish\"}[10m]) + 0.001)","refId":"A"}],"title":"任务失败率（10分钟）","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":12},"id":9,"options":{"displayMode":"gradient","minVizHeight":10,"minVizWidth":0,"orientation":"horizontal","reduceOptions":{"values":false,"calcs":["lastNotNull"],"fields":""},"showUnfilled":true},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_task_execution_count_by_type_total > 0","legendFormat":"{{task_type}}","refId":"A"}],"title":"任务类型分布","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":12},"id":10,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_workflow_instance_running","legendFormat":"运行中","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_workflow_instance_count_total{state=\"success\"}[5m])","legendFormat":"成功-{{process_definition_code}}","refId":"B"}],"title":"工作流实例趋势","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"内存使用率"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.axisPlacement","value":"right"}]},{"matcher":{"id":"byName","options":"CPU使用率"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.axisPlacement","value":"right"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":20},"id":11,"options":{"legend":{"calcs":["last","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_worker_active_execute_thread","legendFormat":"活跃线程数","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_worker_memory_usage","legendFormat":"内存使用率","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"ds_worker_cpu_usage","legendFormat":"CPU使用率","refId":"C"}],"title":"Worker 资源监控","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"short"},"overrides":[{"matcher":{"id":"byName","options":"失败"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":20},"id":12,"options":{"legend":{"calcs":["last","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_alert_send_count_total{status=\"success\"}[5m])","legendFormat":"成功","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"increase(ds_alert_send_count_total{status=\"fail\"}[5m])","legendFormat":"失败","refId":"B"}],"title":"告警发送趋势（5分钟增量）","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":28},"id":13,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(http_server_requests_seconds_sum{uri=~\"/projects.*\"}[5m]) / rate(http_server_requests_seconds_count{uri=~\"/projects.*\"}[5m])","legendFormat":"{{uri}} - {{method}}","refId":"A"}],"title":"API 平均响应时间","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"tooltip":false,"viz":false,"legend":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"reqps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":28},"id":14,"options":{"legend":{"calcs":["mean","max"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"9.5.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(http_server_requests_seconds_count{uri=~\"/projects.*\"}[5m])","legendFormat":"{{uri}} - {{method}}","refId":"A"}],"title":"API 请求速率","type":"timeseries"}],"refresh":"30s","schemaVersion":38,"style":"dark","tags":["dolphinscheduler","大数据"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"includeAll":false,"label":"数据源","multi":false,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"browser","title":"DolphinScheduler 监控面板","uid":"dolphinscheduler-overview","version":1,"weekStart":""}