[Monitoring] Remove staging and prod probe alerts (#3642)

Author: Ram81

monitoring/prometheus/prometheus_production.yml

@@ -22,11 +22,6 @@ scrape_configs:
     static_configs:
       - targets: ['eval.ai']
 
-  - job_name: 'evalai'
-    metrics_path: '/'
-    static_configs:
-      - targets: ['eval.ai']
-
 alerting:
   alertmanagers:
   - path_prefix: '/alert_manager'

monitoring/prometheus/prometheus_staging.yml

@@ -22,11 +22,6 @@ scrape_configs:
     static_configs:
       - targets: ['staging.eval.ai']
 
-  - job_name: 'evalai'
-    metrics_path: '/'
-    static_configs:
-      - targets: ['staging.eval.ai']
-
 alerting:
   alertmanagers:
   - path_prefix: '/alert_manager'

monitoring/prometheus/rules.yml

@@ -23,26 +23,14 @@ groups:
       severity: major
       group: 'instance'
 
-- name: EvalAI-Instance-Status
-  rules:
-  - alert: InstanceDown
-    expr: up{job="evalai"} == 0
-    for: 5m
-    annotations:
-      title: "EvalAI is down"
-      description: "*{{ $labels.job }}* on *{{ $labels.instance }}* has been down for more than 5 minutes"
-    labels:
-      severity: major
-      group: 'instance'
-
 - name: Worker-Down
   rules:
   - alert: WorkerDown
-    expr: ((count by (queue_name) (num_submissions_in_queue{is_remote="0"})) - (count by (queue_name) (num_processed_submissions{is_remote="0"})))  >= 5
+    expr: ((sum by (queue_name) (num_submissions_in_queue{is_remote="0"})) - (sum by (queue_name) (num_processed_submissions{is_remote="0"}))) or sum by(queue_name)(num_submissions_in_queue{is_remote="0"})  >= 5
     for: 120m
     annotations:
       title: "Worker is down"
-      description: "*{{ $labels.queue_name }}* worker is not processing submissions"
+      description: "*{{ $labels.queue_name }}* worker on *{{ $labels.instance }}* is not processing submissions"
     labels:
       severity: major
       group: 'queue_name'