feat: Pushing grafana annotations on incident

README.md

@@ -107,6 +107,7 @@ Follow [these instructions](./docs/slack_app_create.md) to create a new Slack Ap
 | `INCIDENT_CHANNEL_ID`  | When an incident is declared, a 'headline' post is sent to a central channel.<br /><br />See the [demo app settings](./demo/demo/settings/dev.py) for an example of how to get the incident channel ID from the Slack API. |
 | `INCIDENT_BOT_ID`  | We want to invite the Bot to all Incident Channels, so need to know its ID.<br /><br />See the [demo app settings](./demo/demo/settings/dev.py) for an example of how to get the bot ID from the Slack API. |
 | `SLACK_CLIENT`  | Response needs a shared global instance of a Slack Client to talk to the Slack API. Typically this does not require any additional configuration. <br /><pre>from response.slack.client import SlackClient<br />SLACK_CLIENT = SlackClient(SLACK_TOKEN)</pre> |
+|  `GRAFANA_URL`<br /> `GRAFANA_TOKEN`| (OPTIONAL) Send annotations to grafana<br />See [grafana annotations support](./docs/grafana_annoations_support.md). |
 
 ## 3. Running the server
 

demo/demo/settings/base.py

@@ -15,6 +15,7 @@
 
 from django.core.exceptions import ImproperlyConfigured
 
+from response.grafana.client import GrafanaClient
 from response.slack.client import SlackClient
 
 logger = logging.getLogger(__name__)
@@ -197,3 +198,8 @@ def get_env_var(setting, warn_only=False):
 # Whether to use https://pypi.org/project/bleach/ to strip potentially dangerous
 # HTML input in string fields
 RESPONSE_SANITIZE_USER_INPUT = True
+
+GRAFANA_URL = get_env_var("GRAFANA_URL", warn_only=True)
+GRAFANA_CLIENT = None
+if GRAFANA_URL:
+    GRAFANA_CLIENT = GrafanaClient(GRAFANA_URL, get_env_var("GRAFANA_TOKEN"))

docs/grafana_annoations_support.md

@@ -0,0 +1,30 @@
+# Grafana annotation support
+
+<p align="center">
+  <img width="600px" src="./grafana-3.png">
+</p>
+
+## Respone configuration
+To add an incident annotation on your diagrams (see exemple above), please set the following variables:
+
+* `GRAFANA_URL`: The URL of your grafana instance (ex: https://grafana.example.net);
+* `GRAFANA_TOKEN`: A Grafana API key with Editor role.
+
+## Grafana configuration
+
+Edit your dashboard settings.
+
+Go to the "Annotations" section:
+
+![dashboard-edit](./grafana-1.png)
+
+Then add a new annotation with the following settings:
+
+* Name: Incident (for example)
+* Data source: -- Grafana --
+* Color: Enabled :heavy_check_mark: (OPTIONAL)
+* Filter by: Tags
+* Tags:
+  * incident
+
+![annotations setup](./grafana-2.png)

response/apps.py

@@ -18,6 +18,8 @@ def ready(self):
 
         from .core import signals as core_signals  # noqa: F401
 
+        from .grafana import signals as grafana_signals  # noqa: F401
+
         site_settings.RESPONSE_LOGIN_REQUIRED = getattr(
             site_settings, "RESPONSE_LOGIN_REQUIRED", True
         )

response/core/models/incident.py

@@ -77,6 +77,8 @@ class Incident(models.Model):
         max_length=10, blank=True, null=True, choices=SEVERITIES
     )
 
+    grafana_annotation_id = models.PositiveIntegerField(null=True, blank=True)
+
     def __str__(self):
         return self.report
 

response/grafana/client.py

@@ -0,0 +1,75 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class GrafanaError(Exception):
+    def __init__(self, message, grafana_error=None):
+        self.message = message
+        self.grafana_error = grafana_error
+
+
+class GrafanaClient(object):
+    def __init__(self, url, token):
+        self.url = url
+        self.token = token
+
+    def create_annotation(self, **kwargs):
+        logger.info("Create Annotation")
+
+        payload = {
+            "time": kwargs.get("time"),
+            "tags": kwargs.get("tags"),
+            "text": kwargs.get("text"),
+        }
+
+        headers = {
+            "Authorization": "Bearer {}".format(self.token),
+            "Content-Type": "application/json",
+        }
+        res = requests.post(
+            "{0}/api/{1}".format(self.url, "annotations"),
+            headers=headers,
+            json=payload,
+            verify=True,
+        )
+
+        result = res.json()
+        res.close()
+        if res.status_code == 200:
+            return result
+        raise GrafanaError(f"Failed to create annotations '{result}'")
+
+    def update_annotation(self, annotation_id, time, time_end, text, tags):
+        logger.info(f"Update Annotation: '{annotation_id}'")
+
+        payload = {}
+        if time:
+            payload["time"] = int(time.timestamp() * 1000)
+        if time_end:
+            payload["timeEnd"] = int(time_end.timestamp() * 1000)
+        if text:
+            payload["text"] = text
+        if tags:
+            payload["tags"] = tags
+
+        headers = {
+            "Authorization": "Bearer {}".format(self.token),
+            "Content-Type": "application/json",
+        }
+        res = requests.patch(
+            "{0}/api/{1}/{2}".format(self.url, "annotations", annotation_id),
+            headers=headers,
+            json=payload,
+            verify=True,
+        )
+
+        result = res.json()
+        res.close()
+        if res.status_code == 200:
+            return result
+        raise GrafanaError(
+            f"Failed to update annotation: '{annotation_id}': '{result}'"
+        )

response/grafana/signals.py

@@ -0,0 +1,52 @@
+import logging
+
+from django.conf import settings
+from django.db.models.signals import post_save, pre_save
+from django.dispatch import receiver
+
+from response.core.models import Incident
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=Incident)
+def update_grafana_annotation_after_incident_save(sender, instance: Incident, **kwargs):
+    """
+    Reflect changes to incidents in the grafana annotation
+
+    Important: this is called in the synchronous /incident flow so must remain fast (<2 secs)
+
+    """
+    if settings.GRAFANA_CLIENT and instance.grafana_annotation_id:
+        tags = ["incident", instance.severity_text()]
+        text = f"{instance.report} \n {instance.summary}"
+
+        settings.GRAFANA_CLIENT.update_annotation(
+            instance.grafana_annotation_id,
+            time=instance.report_time,
+            time_end=instance.end_time,
+            text=text,
+            tags=tags,
+        )
+
+
+@receiver(pre_save, sender=Incident)
+def create_grafana_annotation_before_incident_save(
+    sender, instance: Incident, **kwargs
+):
+    """
+    Create a grafana annotation ticket before saving the incident
+
+    Important: this is called in the synchronous /incident flow so must remain fast (<2 secs)
+
+    """
+
+    if settings.GRAFANA_CLIENT and not instance.grafana_annotation_id:
+        tags = ["incident", instance.severity_text()]
+        text = f"{instance.report} \n {instance.summary}"
+        start_time = int(instance.report_time.timestamp() * 1000)
+
+        grafana_annotation = settings.GRAFANA_CLIENT.create_annotation(
+            time=start_time, tags=tags, text=text
+        )
+        instance.grafana_annotation_id = grafana_annotation["id"]

response/migrations/0018_incident_grafana_annotation_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2021-04-28 09:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("response", "0017_externaluser_deleted"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="incident",
+            name="grafana_annotation_id",
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+    ]