diff --git a/CHANGELOG.md b/CHANGELOG.md index fd7332e3e1..3546a5a504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## v1.0.10 (2022-07-22) +- Speed-up of alert group web caching +- Internal api for OnCall shifts + ## v1.0.9 (2022-07-21) - Frontend bug fixes & improvements - Support regex_replace() in templates diff --git a/engine/apps/alerts/models/alert_receive_channel.py b/engine/apps/alerts/models/alert_receive_channel.py index 308686c020..2f0cc01666 100644 --- a/engine/apps/alerts/models/alert_receive_channel.py +++ b/engine/apps/alerts/models/alert_receive_channel.py @@ -694,14 +694,19 @@ def listen_for_alertreceivechannel_model_save(sender, instance, created, *args, instance.organization, None, OrganizationLogType.TYPE_HEARTBEAT_CREATED, description ) else: - logger.info(f"Drop AG cache. Reason: save alert_receive_channel {instance.pk}") if kwargs is not None: if "update_fields" in kwargs: if kwargs["update_fields"] is not None: + fields_to_not_to_invalidate_cache = [ + "rate_limit_message_task_id", + "rate_limited_in_slack_at", + "reason_to_skip_escalation", + ] # Hack to not to invalidate web cache on AlertReceiveChannel.start_send_rate_limit_message_task - if "rate_limit_message_task_id" in kwargs["update_fields"]: - return - + for f in fields_to_not_to_invalidate_cache: + if f in kwargs["update_fields"]: + return + logger.info(f"Drop AG cache. Reason: save alert_receive_channel {instance.pk}") invalidate_web_cache_for_alert_group.apply_async(kwargs={"channel_pk": instance.pk}) if instance.integration == AlertReceiveChannel.INTEGRATION_GRAFANA_ALERTING: diff --git a/engine/apps/alerts/tests/test_escalation_policy_snapshot.py b/engine/apps/alerts/tests/test_escalation_policy_snapshot.py index 9a555c35e7..6a4e1630e2 100644 --- a/engine/apps/alerts/tests/test_escalation_policy_snapshot.py +++ b/engine/apps/alerts/tests/test_escalation_policy_snapshot.py @@ -170,8 +170,10 @@ def test_escalation_step_notify_on_call_schedule( schedule = make_schedule(organization, schedule_class=OnCallScheduleCalendar) # create on_call_shift with user to notify + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": timezone.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=7200), } on_call_shift = make_on_call_shift( @@ -216,8 +218,10 @@ def test_escalation_step_notify_on_call_schedule_viewer_user( schedule = make_schedule(organization, schedule_class=OnCallScheduleCalendar) # create on_call_shift with user to notify + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": timezone.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=7200), } on_call_shift = make_on_call_shift( diff --git a/engine/apps/alerts/tests/test_terraform_renderer.py b/engine/apps/alerts/tests/test_terraform_renderer.py index 78ece24b93..1661d89c0d 100644 --- a/engine/apps/alerts/tests/test_terraform_renderer.py +++ b/engine/apps/alerts/tests/test_terraform_renderer.py @@ -99,6 +99,7 @@ def test_render_terraform_file( interval=1, week_start=CustomOnCallShift.MONDAY, start=dateparse.parse_datetime("2021-08-16T17:00:00"), + rotation_start=dateparse.parse_datetime("2021-08-16T17:00:00"), duration=timezone.timedelta(seconds=3600), by_day=["MO", "SA"], rolling_users=[{user.pk: user.public_primary_key}], diff --git a/engine/apps/api/serializers/on_call_shifts.py b/engine/apps/api/serializers/on_call_shifts.py new file mode 100644 index 0000000000..794c466f13 --- /dev/null +++ b/engine/apps/api/serializers/on_call_shifts.py @@ -0,0 +1,203 @@ +from rest_framework import serializers + +from apps.schedules.models import CustomOnCallShift, OnCallScheduleWeb +from apps.user_management.models import User +from common.api_helpers.custom_fields import ( + OrganizationFilteredPrimaryKeyRelatedField, + RollingUsersField, + UsersFilteredByOrganizationField, +) +from common.api_helpers.mixins import EagerLoadingMixin +from common.api_helpers.utils import CurrentOrganizationDefault + + +class OnCallShiftSerializer(EagerLoadingMixin, serializers.ModelSerializer): + id = serializers.CharField(read_only=True, source="public_primary_key") + organization = serializers.HiddenField(default=CurrentOrganizationDefault()) + type = serializers.ChoiceField( + required=True, + choices=CustomOnCallShift.WEB_TYPES, + ) + schedule = OrganizationFilteredPrimaryKeyRelatedField(queryset=OnCallScheduleWeb.objects) + frequency = serializers.ChoiceField(required=False, choices=CustomOnCallShift.FREQUENCY_CHOICES, allow_null=True) + shift_start = serializers.DateTimeField(source="start") + shift_end = serializers.SerializerMethodField() + by_day = serializers.ListField(required=False, allow_null=True) + rolling_users = RollingUsersField( + allow_null=True, + required=False, + child=UsersFilteredByOrganizationField( + queryset=User.objects, required=False, allow_null=True + ), # todo: filter by team? + ) + + class Meta: + model = CustomOnCallShift + fields = [ + "id", + "organization", + "name", + "type", + "schedule", + "priority_level", + "shift_start", + "shift_end", + "rotation_start", + "until", + "frequency", + "interval", + "by_day", + "source", + "rolling_users", + ] + extra_kwargs = { + "interval": {"required": False, "allow_null": True}, + "source": {"required": False, "write_only": True}, + } + + SELECT_RELATED = ["schedule"] + + def get_shift_end(self, obj): + return obj.start + obj.duration + + def to_internal_value(self, data): + data["source"] = CustomOnCallShift.SOURCE_WEB + data["week_start"] = CustomOnCallShift.MONDAY + if not data.get("shift_end"): + raise serializers.ValidationError({"shift_end": ["This field is required."]}) + + result = super().to_internal_value(data) + return result + + def to_representation(self, instance): + result = super().to_representation(instance) + return result + + def validate_name(self, name): + organization = self.context["request"].auth.organization + if name is None: + return name + try: + obj = CustomOnCallShift.objects.get(organization=organization, name=name) + except CustomOnCallShift.DoesNotExist: + return name + if self.instance and obj.id == self.instance.id: + return name + else: + raise serializers.ValidationError(["On-call shift with this name already exists"]) + + def validate_by_day(self, by_day): + if by_day: + for day in by_day: + if day not in CustomOnCallShift.WEB_WEEKDAY_MAP: + raise serializers.ValidationError(["Invalid day value."]) + return by_day + + def validate_interval(self, interval): + if interval is not None: + if not isinstance(interval, int) or interval <= 0: + raise serializers.ValidationError(["Invalid value"]) + return interval + + def validate_rolling_users(self, rolling_users): + result = [] + if rolling_users: + for users in rolling_users: + users_dict = dict() + for user in users: + users_dict[user.pk] = user.public_primary_key + result.append(users_dict) + return result + + def _validate_shift_end(self, start, end): + if end <= start: + raise serializers.ValidationError({"shift_end": ["Incorrect shift end date"]}) + + def _validate_frequency(self, frequency, event_type, rolling_users, interval, by_day): + if frequency is None: + if rolling_users and len(rolling_users) > 1: + raise serializers.ValidationError( + {"rolling_users": ["Cannot set multiple user groups for non-recurrent shifts"]} + ) + if interval is not None: + raise serializers.ValidationError({"interval": ["Cannot set interval for non-recurrent shifts"]}) + if by_day: + raise serializers.ValidationError({"by_day": ["Cannot set days value for non-recurrent shifts"]}) + else: + if event_type == CustomOnCallShift.TYPE_OVERRIDE: + raise serializers.ValidationError( + {"frequency": ["Cannot set 'frequency' for shifts with type 'override'"]} + ) + if frequency != CustomOnCallShift.FREQUENCY_WEEKLY and by_day: + raise serializers.ValidationError({"by_day": ["Cannot set days value for this frequency type"]}) + + def _validate_rotation_start(self, shift_start, rotation_start): + if rotation_start < shift_start: + raise serializers.ValidationError({"rotation_start": ["Incorrect rotation start date"]}) + + def _validate_until(self, rotation_start, until): + if until is not None and until < rotation_start: + raise serializers.ValidationError({"until": ["Incorrect rotation end date"]}) + + def _correct_validated_data(self, event_type, validated_data): + fields_to_update_for_overrides = [ + "priority_level", + "frequency", + "interval", + "by_day", + "until", + "rotation_start", + ] + if event_type == CustomOnCallShift.TYPE_OVERRIDE: + for field in fields_to_update_for_overrides: + value = None + if field == "priority_level": + value = 0 + elif field == "rotation_start": + value = validated_data["start"] + validated_data[field] = value + + self._validate_frequency( + validated_data.get("frequency"), + event_type, + validated_data.get("rolling_users"), + validated_data.get("interval"), + validated_data.get("by_day"), + ) + self._validate_rotation_start(validated_data["start"], validated_data["rotation_start"]) + self._validate_until(validated_data["rotation_start"], validated_data.get("until")) + + # convert shift_end into internal value and validate + raw_shift_end = self.initial_data["shift_end"] + shift_end = serializers.DateTimeField().to_internal_value(raw_shift_end) + self._validate_shift_end(validated_data["start"], shift_end) + + validated_data["duration"] = shift_end - validated_data["start"] + if validated_data.get("schedule"): + validated_data["team"] = validated_data["schedule"].team + + return validated_data + + def create(self, validated_data): + validated_data = self._correct_validated_data(validated_data["type"], validated_data) + + instance = super().create(validated_data) + + instance.start_drop_ical_and_check_schedule_tasks(instance.schedule) + return instance + + +class OnCallShiftUpdateSerializer(OnCallShiftSerializer): + schedule = serializers.CharField(read_only=True, source="schedule.public_primary_key") + type = serializers.ReadOnlyField() + + class Meta(OnCallShiftSerializer.Meta): + read_only_fields = ("schedule", "type") + + def update(self, instance, validated_data): + validated_data = self._correct_validated_data(instance.type, validated_data) + + result = super().update(instance, validated_data) + + instance.start_drop_ical_and_check_schedule_tasks(instance.schedule) + return result diff --git a/engine/apps/api/tests/test_schedules.py b/engine/apps/api/tests/test_schedules.py index cadcb5f46b..8cb0319dea 100644 --- a/engine/apps/api/tests/test_schedules.py +++ b/engine/apps/api/tests/test_schedules.py @@ -397,8 +397,10 @@ def test_events_calendar( name="test_calendar_schedule", ) + start_date = timezone.now().replace(microsecond=0) data = { - "start": timezone.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=7200), "priority_level": 2, } @@ -460,6 +462,7 @@ def test_filter_events_calendar( start_date = now - timezone.timedelta(days=7) data = { "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=7200), "priority_level": 1, "frequency": CustomOnCallShift.FREQUENCY_WEEKLY, @@ -539,6 +542,7 @@ def test_filter_events_range_calendar( start_date = now - timezone.timedelta(days=7) data = { "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=7200), "priority_level": 1, "frequency": CustomOnCallShift.FREQUENCY_WEEKLY, diff --git a/engine/apps/api/urls.py b/engine/apps/api/urls.py index a82ee4e985..1eb9adf2c5 100644 --- a/engine/apps/api/urls.py +++ b/engine/apps/api/urls.py @@ -17,6 +17,7 @@ from .views.integration_heartbeat import IntegrationHeartBeatView from .views.live_setting import LiveSettingViewSet from .views.maintenance import MaintenanceAPIView, MaintenanceStartAPIView, MaintenanceStopAPIView +from .views.on_call_shifts import OnCallShiftView from .views.organization import ( CurrentOrganizationView, GetChannelVerificationCode, @@ -65,6 +66,7 @@ router.register(r"organization_logs", OrganizationLogRecordView, basename="organization_log") router.register(r"tokens", PublicApiTokenView, basename="api_token") router.register(r"live_settings", LiveSettingViewSet, basename="live_settings") +router.register(r"oncall_shifts", OnCallShiftView, basename="oncall_shifts") if settings.MOBILE_APP_PUSH_NOTIFICATIONS_ENABLED: router.register(r"device/apns", APNSDeviceAuthorizedViewSet) diff --git a/engine/apps/api/views/on_call_shifts.py b/engine/apps/api/views/on_call_shifts.py new file mode 100644 index 0000000000..a12e5c0bd6 --- /dev/null +++ b/engine/apps/api/views/on_call_shifts.py @@ -0,0 +1,102 @@ +from django.db.models import Q +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.viewsets import ModelViewSet + +from apps.api.permissions import MODIFY_ACTIONS, READ_ACTIONS, ActionPermission, AnyRole, IsAdmin +from apps.api.serializers.on_call_shifts import OnCallShiftSerializer, OnCallShiftUpdateSerializer +from apps.auth_token.auth import PluginAuthentication +from apps.schedules.models import CustomOnCallShift +from apps.user_management.organization_log_creator import OrganizationLogType, create_organization_log +from common.api_helpers.mixins import PublicPrimaryKeyMixin, UpdateSerializerMixin +from common.api_helpers.paginators import FiftyPageSizePaginator + + +class OnCallShiftView(PublicPrimaryKeyMixin, UpdateSerializerMixin, ModelViewSet): + authentication_classes = (PluginAuthentication,) + permission_classes = (IsAuthenticated, ActionPermission) + + action_permissions = { + IsAdmin: MODIFY_ACTIONS, + AnyRole: (*READ_ACTIONS, "details", "frequency_options", "days_options"), + } + + model = CustomOnCallShift + serializer_class = OnCallShiftSerializer + update_serializer_class = OnCallShiftUpdateSerializer + + pagination_class = FiftyPageSizePaginator + + filter_backends = [DjangoFilterBackend] + + def get_queryset(self): + schedule_id = self.request.query_params.get("schedule_id", None) + lookup_kwargs = Q() + if schedule_id: + lookup_kwargs = Q( + Q(schedule__public_primary_key=schedule_id) | Q(schedules__public_primary_key=schedule_id) + ) + + queryset = CustomOnCallShift.objects.filter( + lookup_kwargs, + organization=self.request.auth.organization, + team=self.request.user.current_team, + ) + + queryset = self.serializer_class.setup_eager_loading(queryset) + return queryset.order_by("schedules") + + def perform_create(self, serializer): + serializer.save() + instance = serializer.instance + organization = self.request.auth.organization + user = self.request.user + description = ( + f"Custom on-call shift with params: {instance.repr_settings_for_client_side_logging} " + f"was created" # todo + ) + create_organization_log(organization, user, OrganizationLogType.TYPE_ON_CALL_SHIFT_CREATED, description) + + def perform_update(self, serializer): + organization = self.request.auth.organization + user = self.request.user + old_state = serializer.instance.repr_settings_for_client_side_logging + serializer.save() + new_state = serializer.instance.repr_settings_for_client_side_logging + description = f"Settings of custom on-call shift was changed " f"from:\n{old_state}\nto:\n{new_state}" + create_organization_log(organization, user, OrganizationLogType.TYPE_ON_CALL_SHIFT_CHANGED, description) + + def perform_destroy(self, instance): + organization = self.request.auth.organization + user = self.request.user + description = ( + f"Custom on-call shift " f"with params: {instance.repr_settings_for_client_side_logging} was deleted" + ) + create_organization_log(organization, user, OrganizationLogType.TYPE_ON_CALL_SHIFT_DELETED, description) + instance.delete() + + @action(detail=False, methods=["get"]) + def frequency_options(self, request): + return Response( + [ + { + "display_name": display_name, + "value": freq, + } + for freq, display_name in CustomOnCallShift.WEB_FREQUENCY_CHOICES_MAP.items() + ] + ) + + @action(detail=False, methods=["get"]) + def days_options(self, request): + return Response( + [ + { + "display_name": display_name, + "value": value, + } + for value, display_name in CustomOnCallShift.WEB_WEEKDAY_MAP.items() + ] + ) diff --git a/engine/apps/public_api/serializers/on_call_shifts.py b/engine/apps/public_api/serializers/on_call_shifts.py index 1fd85aa656..d65928038a 100644 --- a/engine/apps/public_api/serializers/on_call_shifts.py +++ b/engine/apps/public_api/serializers/on_call_shifts.py @@ -3,24 +3,17 @@ from rest_framework import fields, serializers from apps.schedules.models import CustomOnCallShift -from apps.schedules.tasks import ( - drop_cached_ical_task, - schedule_notify_about_empty_shifts_in_schedule, - schedule_notify_about_gaps_in_schedule, -) from apps.user_management.models import User -from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField, UsersFilteredByOrganizationField +from common.api_helpers.custom_fields import ( + RollingUsersField, + TeamPrimaryKeyRelatedField, + UsersFilteredByOrganizationField, +) from common.api_helpers.exceptions import BadRequest from common.api_helpers.mixins import EagerLoadingMixin from common.api_helpers.utils import CurrentOrganizationDefault -class RollingUsersField(serializers.ListField): - def to_representation(self, value): - result = [list(d.values()) for d in value] - return result - - class CustomOnCallShiftTypeField(fields.CharField): def to_representation(self, value): return CustomOnCallShift.PUBLIC_TYPE_CHOICES_MAP[value] @@ -90,6 +83,7 @@ class CustomOnCallShiftSerializer(EagerLoadingMixin, serializers.ModelSerializer required=False, child=UsersFilteredByOrganizationField(queryset=User.objects, required=False, allow_null=True), ) + rotation_start = serializers.DateTimeField(required=False) class Meta: model = CustomOnCallShift @@ -103,6 +97,7 @@ class Meta: "level", "start", "duration", + "rotation_start", "frequency", "interval", "until", @@ -137,7 +132,11 @@ def create(self, validated_data): validated_data.get("by_day"), validated_data.get("by_monthday"), ) + if not validated_data.get("rotation_start"): + validated_data["rotation_start"] = validated_data["start"] instance = super().create(validated_data) + for schedule in instance.schedules.all(): + instance.start_drop_ical_and_check_schedule_tasks(schedule) return instance def validate_name(self, name): @@ -227,6 +226,9 @@ def _validate_start(self, start): def _validate_until(self, until): self._validate_date_format(until) + def _validate_rotation_start(self, rotation_start): + self._validate_date_format(rotation_start) + def to_internal_value(self, data): if data.get("users", []) is None: # terraform case data["users"] = [] @@ -236,6 +238,8 @@ def to_internal_value(self, data): data["source"] = CustomOnCallShift.SOURCE_API if data.get("start") is not None: self._validate_start(data["start"]) + if data.get("rotation_start") is not None: + self._validate_rotation_start(data["rotation_start"]) if data.get("until") is not None: self._validate_until(data["until"]) result = super().to_internal_value(data) @@ -245,6 +249,7 @@ def to_representation(self, instance): result = super().to_representation(instance) result["duration"] = int(instance.duration.total_seconds()) result["start"] = instance.start.strftime("%Y-%m-%dT%H:%M:%S") + result["rotation_start"] = instance.rotation_start.strftime("%Y-%m-%dT%H:%M:%S") if instance.until is not None: result["until"] = instance.until.strftime("%Y-%m-%dT%H:%M:%S") result = self._get_fields_to_represent(instance, result) @@ -310,6 +315,7 @@ def _correct_validated_data(self, event_type, validated_data): "by_monthday", "rolling_users", "start_rotation_from_user_index", + "until", ], } for field in fields_to_update_map[event_type]: @@ -335,6 +341,7 @@ class CustomOnCallShiftUpdateSerializer(CustomOnCallShiftSerializer): duration = serializers.DurationField(required=False) name = serializers.CharField(required=False) start = serializers.DateTimeField(required=False) + rotation_start = serializers.DateTimeField(required=False) team_id = TeamPrimaryKeyRelatedField(read_only=True, source="team") def update(self, instance, validated_data): @@ -356,9 +363,5 @@ def update(self, instance, validated_data): validated_data = self._correct_validated_data(event_type, validated_data) result = super().update(instance, validated_data) for schedule in instance.schedules.all(): - drop_cached_ical_task.apply_async( - (schedule.pk,), - ) - schedule_notify_about_empty_shifts_in_schedule.apply_async((instance.pk,)) - schedule_notify_about_gaps_in_schedule.apply_async((instance.pk,)) + instance.start_drop_ical_and_check_schedule_tasks(schedule) return result diff --git a/engine/apps/public_api/tests/test_on_call_shifts.py b/engine/apps/public_api/tests/test_on_call_shifts.py index 8311c00e2e..2934c6c777 100644 --- a/engine/apps/public_api/tests/test_on_call_shifts.py +++ b/engine/apps/public_api/tests/test_on_call_shifts.py @@ -2,6 +2,7 @@ import pytest from django.urls import reverse +from django.utils import timezone from rest_framework import status from rest_framework.test import APIClient @@ -45,9 +46,11 @@ def test_get_on_call_shift(make_organization_and_user_with_token, make_on_call_s organization, user, token = make_organization_and_user_with_token() client = APIClient() + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": datetime.datetime.now().replace(microsecond=0), - "duration": datetime.timedelta(seconds=7200), + "start": start_date, + "rotation_start": start_date, + "duration": timezone.timedelta(seconds=7200), } schedule = make_schedule(organization, schedule_class=OnCallScheduleCalendar) on_call_shift = make_on_call_shift( @@ -68,6 +71,7 @@ def test_get_on_call_shift(make_organization_and_user_with_token, make_on_call_s "time_zone": None, "level": 0, "start": on_call_shift.start.strftime("%Y-%m-%dT%H:%M:%S"), + "rotation_start": on_call_shift.start.strftime("%Y-%m-%dT%H:%M:%S"), "duration": int(on_call_shift.duration.total_seconds()), "users": [user.public_primary_key], } @@ -82,8 +86,11 @@ def test_get_override_on_call_shift(make_organization_and_user_with_token, make_ client = APIClient() schedule = make_schedule(organization, schedule_class=OnCallScheduleWeb) + + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": datetime.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": datetime.timedelta(seconds=7200), "schedule": schedule, } @@ -101,6 +108,7 @@ def test_get_override_on_call_shift(make_organization_and_user_with_token, make_ "type": "override", "time_zone": None, "start": on_call_shift.start.strftime("%Y-%m-%dT%H:%M:%S"), + "rotation_start": on_call_shift.start.strftime("%Y-%m-%dT%H:%M:%S"), "duration": int(on_call_shift.duration.total_seconds()), "users": [user.public_primary_key], } @@ -125,6 +133,7 @@ def test_create_on_call_shift(make_organization_and_user_with_token): "type": "recurrent_event", "level": 1, "start": start.strftime("%Y-%m-%dT%H:%M:%S"), + "rotation_start": start.strftime("%Y-%m-%dT%H:%M:%S"), "duration": 10800, "users": [user.public_primary_key], "week_start": "MO", @@ -145,6 +154,7 @@ def test_create_on_call_shift(make_organization_and_user_with_token): "time_zone": None, "level": data["level"], "start": data["start"], + "rotation_start": data["rotation_start"], "duration": data["duration"], "frequency": data["frequency"], "interval": data["interval"], @@ -174,6 +184,7 @@ def test_create_override_on_call_shift(make_organization_and_user_with_token): "name": "test name", "type": "override", "start": start.strftime("%Y-%m-%dT%H:%M:%S"), + "rotation_start": start.strftime("%Y-%m-%dT%H:%M:%S"), "duration": 10800, "users": [user.public_primary_key], } @@ -188,6 +199,7 @@ def test_create_override_on_call_shift(make_organization_and_user_with_token): "type": "override", "time_zone": None, "start": data["start"], + "rotation_start": data["rotation_start"], "duration": data["duration"], "users": [user.public_primary_key], } @@ -201,8 +213,10 @@ def test_update_on_call_shift(make_organization_and_user_with_token, make_on_cal organization, user, token = make_organization_and_user_with_token() client = APIClient() + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": datetime.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": datetime.timedelta(seconds=7200), "frequency": CustomOnCallShift.FREQUENCY_WEEKLY, "interval": 2, @@ -237,6 +251,7 @@ def test_update_on_call_shift(make_organization_and_user_with_token, make_on_cal "time_zone": None, "level": 0, "start": on_call_shift.start.strftime("%Y-%m-%dT%H:%M:%S"), + "rotation_start": on_call_shift.rotation_start.strftime("%Y-%m-%dT%H:%M:%S"), "duration": data_to_update["duration"], "frequency": "weekly", "interval": on_call_shift.interval, @@ -275,8 +290,10 @@ def test_update_on_call_shift_invalid_field(make_organization_and_user_with_toke organization, user, token = make_organization_and_user_with_token() client = APIClient() + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": datetime.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": datetime.timedelta(seconds=7200), "frequency": CustomOnCallShift.FREQUENCY_WEEKLY, "interval": 2, @@ -300,8 +317,10 @@ def test_delete_on_call_shift(make_organization_and_user_with_token, make_on_cal organization, user, token = make_organization_and_user_with_token() client = APIClient() + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": datetime.datetime.now().replace(microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": datetime.timedelta(seconds=7200), } on_call_shift = make_on_call_shift( diff --git a/engine/apps/public_api/tests/test_schedules.py b/engine/apps/public_api/tests/test_schedules.py index 52acd29f0e..b772a8bafa 100644 --- a/engine/apps/public_api/tests/test_schedules.py +++ b/engine/apps/public_api/tests/test_schedules.py @@ -294,8 +294,10 @@ def test_update_calendar_schedule_with_custom_event( schedule_class=OnCallScheduleCalendar, channel=slack_channel_id, ) + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": timezone.now().replace(tzinfo=None, microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=10800), } on_call_shift = make_on_call_shift( @@ -347,8 +349,10 @@ def test_update_calendar_schedule_invalid_override( organization, schedule_class=OnCallScheduleCalendar, ) + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": timezone.now().replace(tzinfo=None, microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=10800), } on_call_shift = make_on_call_shift(organization=organization, shift_type=CustomOnCallShift.TYPE_OVERRIDE, **data) @@ -378,8 +382,10 @@ def test_update_web_schedule_with_override( organization, schedule_class=OnCallScheduleWeb, ) + start_date = timezone.datetime.now().replace(microsecond=0) data = { - "start": timezone.now().replace(tzinfo=None, microsecond=0), + "start": start_date, + "rotation_start": start_date, "duration": timezone.timedelta(seconds=10800), } on_call_shift = make_on_call_shift(organization=organization, shift_type=CustomOnCallShift.TYPE_OVERRIDE, **data) diff --git a/engine/apps/schedules/migrations/0006_customoncallshift_rotation_start.py b/engine/apps/schedules/migrations/0006_customoncallshift_rotation_start.py new file mode 100644 index 0000000000..682df51407 --- /dev/null +++ b/engine/apps/schedules/migrations/0006_customoncallshift_rotation_start.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.13 on 2022-07-12 08:03 + +from django.db import migrations, models +from django.db.models import F + + +def fill_rotation_start_field(apps, schema_editor): + CustomOnCallShift = apps.get_model("schedules", "CustomOnCallShift") + CustomOnCallShift.objects.update(rotation_start=F("start")) + + +class Migration(migrations.Migration): + + dependencies = [ + ('schedules', '0005_auto_20220704_1947'), + ] + + operations = [ + migrations.AddField( + model_name='customoncallshift', + name='rotation_start', + field=models.DateTimeField(default=None, null=True), + ), + migrations.RunPython(fill_rotation_start_field, migrations.RunPython.noop), + migrations.AlterField( + model_name='customoncallshift', + name='rotation_start', + field=models.DateTimeField(), + ), + ] diff --git a/engine/apps/schedules/models/custom_on_call_shift.py b/engine/apps/schedules/models/custom_on_call_shift.py index 0f2753da32..64d72ae85c 100644 --- a/engine/apps/schedules/models/custom_on_call_shift.py +++ b/engine/apps/schedules/models/custom_on_call_shift.py @@ -13,7 +13,11 @@ from icalendar.cal import Event from recurring_ical_events import UnfoldableCalendar -from apps.schedules.tasks import drop_cached_ical_task +from apps.schedules.tasks import ( + drop_cached_ical_task, + schedule_notify_about_empty_shifts_in_schedule, + schedule_notify_about_gaps_in_schedule, +) from apps.user_management.models import User from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length @@ -57,6 +61,13 @@ class CustomOnCallShift(models.Model): FREQUENCY_MONTHLY: "monthly", } + WEB_FREQUENCY_CHOICES_MAP = { + FREQUENCY_HOURLY: "hours", + FREQUENCY_DAILY: "days", + FREQUENCY_WEEKLY: "weeks", + FREQUENCY_MONTHLY: "months", + } + ( TYPE_SINGLE_EVENT, TYPE_RECURRENT_EVENT, @@ -78,6 +89,11 @@ class CustomOnCallShift(models.Model): TYPE_OVERRIDE: "override", } + WEB_TYPES = ( + TYPE_ROLLING_USERS_EVENT, + TYPE_OVERRIDE, + ) + (MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7) WEEKDAY_CHOICES = ( @@ -99,6 +115,16 @@ class CustomOnCallShift(models.Model): SATURDAY: "SA", SUNDAY: "SU", } + + WEB_WEEKDAY_MAP = { + "MO": "Monday", + "TU": "Tuesday", + "WE": "Wednesday", + "TH": "Thursday", + "FR": "Friday", + "SA": "Saturday", + "SU": "Sunday", + } ( SOURCE_WEB, SOURCE_API, @@ -151,6 +177,8 @@ class CustomOnCallShift(models.Model): start = models.DateTimeField() # event start datetime duration = models.DurationField() # duration in seconds + rotation_start = models.DateTimeField() # used for calculation users rotation and rotation start date + frequency = models.IntegerField(choices=FREQUENCY_CHOICES, null=True, default=None) priority_level = models.IntegerField(default=0) @@ -173,7 +201,10 @@ class Meta: def delete(self, *args, **kwargs): for schedule in self.schedules.all(): - drop_cached_ical_task.apply_async((schedule.pk,)) + self.start_drop_ical_and_check_schedule_tasks(schedule) + if self.schedule: + self.start_drop_ical_and_check_schedule_tasks(self.schedule) + # todo: add soft delete super().delete(*args, **kwargs) @property @@ -205,7 +236,8 @@ def repr_settings_for_client_side_logging(self) -> str: result += ( f", frequency: {self.get_frequency_display()}, interval: {self.interval}, " f"week start: {self.week_start}, by day: {self.by_day}, by month: {self.by_month}, " - f"by monthday: {self.by_monthday}, until: {self.until.isoformat() if self.until else None}" + f"by monthday: {self.by_monthday}, rotation start: {self.rotation_start.isoformat()}, " + f"until: {self.until.isoformat() if self.until else None}" ) return result @@ -219,6 +251,8 @@ def convert_to_ical(self, time_zone="UTC"): users_queue = self.get_rolling_users() for counter, users in enumerate(users_queue, start=1): start = self.get_next_start_date(event_ical) + if not start: # means that rotation ends before next event starts + break for user_counter, user in enumerate(users, start=1): event_ical = self.generate_ical(user, start, user_counter, counter, time_zone) result += event_ical @@ -280,6 +314,10 @@ def get_next_start_date(self, event_ical): if days_for_next_event > DAYS_IN_A_MONTH: days_for_next_event = days_for_next_event % DAYS_IN_A_MONTH next_event_start = current_event_start + timezone.timedelta(days=days_for_next_event) + + # check if rotation ends before next event starts + if self.until and next_event_start > self.until: + return next_event = None # repetitions generate the next event shift according with the recurrence rules repetitions = UnfoldableCalendar(current_event).RepeatedEvent( @@ -356,3 +394,8 @@ def add_rolling_users(self, rolling_users_list): result.append({user.pk: user.public_primary_key for user in users}) self.rolling_users = result self.save(update_fields=["rolling_users"]) + + def start_drop_ical_and_check_schedule_tasks(self, schedule): + drop_cached_ical_task.apply_async((schedule.pk,)) + schedule_notify_about_empty_shifts_in_schedule.apply_async((schedule.pk,)) + schedule_notify_about_gaps_in_schedule.apply_async((schedule.pk,)) diff --git a/engine/apps/schedules/tests/test_custom_on_call_shift.py b/engine/apps/schedules/tests/test_custom_on_call_shift.py index 8e507c89a9..1782e48350 100644 --- a/engine/apps/schedules/tests/test_custom_on_call_shift.py +++ b/engine/apps/schedules/tests/test_custom_on_call_shift.py @@ -15,6 +15,7 @@ def test_get_on_call_users_from_single_event(make_organization_and_user, make_on data = { "priority_level": 1, "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), } @@ -41,6 +42,7 @@ def test_get_on_call_users_from_web_schedule_override(make_organization_and_user data = { "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), "schedule": schedule, } @@ -65,6 +67,7 @@ def test_get_on_call_users_from_recurrent_event(make_organization_and_user, make data = { "priority_level": 1, "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), "frequency": CustomOnCallShift.FREQUENCY_DAILY, "interval": 2, @@ -107,6 +110,7 @@ def test_get_on_call_users_from_web_schedule_recurrent_event( data = { "priority_level": 1, "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), "frequency": CustomOnCallShift.FREQUENCY_DAILY, "interval": 2, @@ -149,6 +153,7 @@ def test_get_on_call_users_from_rolling_users_event( data = { "priority_level": 1, "start": now, + "rotation_start": now, "duration": timezone.timedelta(seconds=10800), "frequency": CustomOnCallShift.FREQUENCY_DAILY, "interval": 2, @@ -221,6 +226,7 @@ def test_get_oncall_users_for_multiple_schedules( shift_type=CustomOnCallShift.TYPE_SINGLE_EVENT, priority_level=1, start=now, + rotation_start=now, duration=timezone.timedelta(minutes=30), ) @@ -229,6 +235,7 @@ def test_get_oncall_users_for_multiple_schedules( shift_type=CustomOnCallShift.TYPE_SINGLE_EVENT, priority_level=1, start=now, + rotation_start=now, duration=timezone.timedelta(minutes=10), ) @@ -237,6 +244,7 @@ def test_get_oncall_users_for_multiple_schedules( shift_type=CustomOnCallShift.TYPE_SINGLE_EVENT, priority_level=1, start=now + timezone.timedelta(minutes=10), + rotation_start=now + timezone.timedelta(minutes=10), duration=timezone.timedelta(minutes=30), ) @@ -275,6 +283,7 @@ def test_shift_convert_to_ical(make_organization_and_user, make_on_call_shift): data = { "priority_level": 1, "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), "frequency": CustomOnCallShift.FREQUENCY_HOURLY, "interval": 1, diff --git a/engine/apps/schedules/tests/test_ical_utils.py b/engine/apps/schedules/tests/test_ical_utils.py index 2a737df2d4..e7946617d0 100644 --- a/engine/apps/schedules/tests/test_ical_utils.py +++ b/engine/apps/schedules/tests/test_ical_utils.py @@ -41,6 +41,7 @@ def test_list_users_to_notify_from_ical_viewers_inclusion( data = { "priority_level": 1, "start": date, + "rotation_start": date, "duration": timezone.timedelta(seconds=10800), } on_call_shift = make_on_call_shift( diff --git a/engine/apps/user_management/models/user.py b/engine/apps/user_management/models/user.py index ca769243ac..6e8b27dcb2 100644 --- a/engine/apps/user_management/models/user.py +++ b/engine/apps/user_management/models/user.py @@ -8,7 +8,6 @@ from django.dispatch import receiver from emoji import demojize -from apps.alerts.tasks import invalidate_web_cache_for_alert_group from apps.schedules.tasks import drop_cached_ical_for_custom_events_for_organization from common.constants.role import Role from common.public_primary_keys import generate_public_primary_key, increase_public_primary_key_length @@ -255,14 +254,6 @@ def short(self): # TODO: check whether this signal can be moved to save method of the model @receiver(post_save, sender=User) def listen_for_user_model_save(sender, instance, created, *args, **kwargs): - # if kwargs is not None: - # if "update_fields" in kwargs: - # if kwargs["update_fields"] is not None: - # if "username" not in kwargs["update_fields"]: - # return - drop_cached_ical_for_custom_events_for_organization.apply_async( (instance.organization_id,), ) - logger.info(f"Drop AG cache. Reason: save user {instance.pk}") - invalidate_web_cache_for_alert_group.apply_async(kwargs={"org_pk": instance.organization_id}) diff --git a/engine/common/api_helpers/custom_fields.py b/engine/common/api_helpers/custom_fields.py index 597aff5251..79b96012e8 100644 --- a/engine/common/api_helpers/custom_fields.py +++ b/engine/common/api_helpers/custom_fields.py @@ -171,3 +171,9 @@ def to_representation(self, value): if value is not None: return value.public_primary_key return value + + +class RollingUsersField(serializers.ListField): + def to_representation(self, value): + result = [list(d.values()) for d in value] + return result