Added preview and migration API endpoints for route migration from regex into jinja2

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added preview and migration API endpoints for route migration from regex into jinja2 ([1715](https://github.com/grafana/oncall/pull/1715))
 - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386))
 
 ## v1.2.11 (2023-04-14)

engine/apps/api/serializers/channel_filter.py

@@ -26,6 +26,7 @@ class ChannelFilterSerializer(OrderedModelSerializerMixin, EagerLoadingMixin, se
         queryset=TelegramToOrganizationConnector.objects, filter_field="organization", allow_null=True, required=False
     )
     order = serializers.IntegerField(required=False)
+    filtering_term_as_jinja2 = serializers.SerializerMethodField()
 
     SELECT_RELATED = ["escalation_chain", "alert_receive_channel"]
 
@@ -45,6 +46,7 @@ class Meta:
             "notify_in_slack",
             "notify_in_telegram",
             "notification_backends",
+            "filtering_term_as_jinja2",
         ]
         read_only_fields = ["created_at", "is_default"]
         extra_kwargs = {"filtering_term": {"required": True, "allow_null": False}}
@@ -107,6 +109,16 @@ def validate_notification_backends(self, notification_backends):
             notification_backends = updated
         return notification_backends
 
+    def get_filtering_term_as_jinja2(self, obj):
+        """
+        Returns the regex filtering term as a jinja2, for the preview before migration from regex to jinja2"""
+        if obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_JINJA2:
+            return obj.filtering_term
+        elif obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX:
+            # Four curly braces will result in two curly braces in the final string
+            # rf"..." is a raw f string, to keep original filtering_term
+            return rf'{{{{ payload | json_dumps | regex_search("{obj.filtering_term}") }}}}'
+
 
 class ChannelFilterCreateSerializer(ChannelFilterSerializer):
     alert_receive_channel = OrganizationFilteredPrimaryKeyRelatedField(queryset=AlertReceiveChannel.objects)

engine/apps/api/tests/test_channel_filter.py

@@ -502,3 +502,67 @@ def test_channel_filter_update_invalid_notification_backends(
     assert response.status_code == status.HTTP_400_BAD_REQUEST
     assert response.json() == {"notification_backends": ["Invalid messaging backend"]}
     assert channel_filter.notification_backends is None
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "role,expected_status",
+    [
+        (LegacyAccessControlRole.ADMIN, status.HTTP_200_OK),
+        (LegacyAccessControlRole.EDITOR, status.HTTP_403_FORBIDDEN),
+        (LegacyAccessControlRole.VIEWER, status.HTTP_403_FORBIDDEN),
+    ],
+)
+def test_channel_filter_convert_from_regex_to_jinja2(
+    make_organization_and_user_with_plugin_token,
+    make_alert_receive_channel,
+    make_channel_filter,
+    make_user_auth_headers,
+    role,
+    expected_status,
+):
+    organization, user, token = make_organization_and_user_with_plugin_token(role)
+    alert_receive_channel = make_alert_receive_channel(organization)
+
+    make_channel_filter(alert_receive_channel, is_default=True)
+
+    # r"..." used to keep this string as raw string
+    regex_filtering_term = r"\".*\": \"This alert was sent by user for the demonstration purposes\""
+    final_filtering_term = r'{{ payload | json_dumps | regex_search("\".*\": \"This alert was sent by user for the demonstration purposes\"") }}'
+    payload = {"description": "This alert was sent by user for the demonstration purposes"}
+
+    regex_channel_filter = make_channel_filter(
+        alert_receive_channel,
+        filtering_term=regex_filtering_term,
+        is_default=False,
+    )
+    # Check if the filtering term is a regex
+    assert regex_channel_filter.filtering_term_type == regex_channel_filter.FILTERING_TERM_TYPE_REGEX
+    # Check if the alert is matched to the channel filter (route) regex
+    assert bool(regex_channel_filter.is_satisfying(payload)) is True
+
+    client = APIClient()
+    url = reverse("api-internal:channel_filter-detail", kwargs={"pk": regex_channel_filter.public_primary_key})
+
+    response = client.get(url, format="json", **make_user_auth_headers(user, token))
+
+    assert response.status_code == status.HTTP_200_OK
+    # Check if preview of the filtering term migration is correct
+    assert response.json()["filtering_term_as_jinja2"] == final_filtering_term
+
+    url = reverse(
+        "api-internal:channel_filter-convert-from-regex-to-jinja2",
+        kwargs={"pk": regex_channel_filter.public_primary_key},
+    )
+    response = client.post(url, **make_user_auth_headers(user, token))
+    # Only admins can convert from regex to jinja2
+    assert response.status_code == expected_status
+    if expected_status == status.HTTP_200_OK:
+        regex_channel_filter.refresh_from_db()
+        # Regex is now converted to jinja2
+        jinja2_channel_filter = regex_channel_filter
+        # Check if the filtering term is a jinja2, and if it is correct
+        assert jinja2_channel_filter.filtering_term_type == jinja2_channel_filter.FILTERING_TERM_TYPE_JINJA2
+        assert jinja2_channel_filter.filtering_term == final_filtering_term
+        # Check if the same alert is matched to the channel filter (route) new jinja2
+        assert bool(jinja2_channel_filter.is_satisfying(payload)) is True

engine/apps/api/views/channel_filter.py

@@ -41,6 +41,7 @@ class ChannelFilterView(
         "destroy": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
         "move_to_position": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
         "send_demo_alert": [RBACPermission.Permissions.INTEGRATIONS_TEST],
+        "convert_from_regex_to_jinja2": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
     }
 
     model = ChannelFilter
@@ -143,3 +144,16 @@ def send_demo_alert(self, request, pk):
         except UnableToSendDemoAlert as e:
             raise BadRequest(detail=str(e))
         return Response(status=status.HTTP_200_OK)
+
+    @action(detail=True, methods=["post"])
+    def convert_from_regex_to_jinja2(self, request, pk):
+        instance = self.get_queryset().get(public_primary_key=pk)
+        if not instance.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX:
+            raise BadRequest(detail="Only regex filtering term type is supported")
+
+        serializer_class = self.serializer_class
+
+        instance.filtering_term = serializer_class(instance).get_filtering_term_as_jinja2(instance)
+        instance.filtering_term_type = ChannelFilter.FILTERING_TERM_TYPE_JINJA2
+        instance.save()
+        return Response(status=status.HTTP_200_OK, data=serializer_class(instance).data)

engine/common/jinja_templater/filters.py

@@ -37,3 +37,17 @@ def regex_match(pattern, value):
         return bool(re.match(value, pattern))
     except (ValueError, AttributeError, TypeError):
         return None
+
+
+def regex_search(pattern, value):
+    try:
+        return bool(re.search(value, pattern))
+    except (ValueError, AttributeError, TypeError):
+        return None
+
+
+def json_dumps(value):
+    try:
+        return json.dumps(value)
+    except (ValueError, AttributeError, TypeError):
+        return None

engine/common/jinja_templater/jinja_template_env.py

@@ -3,7 +3,15 @@
 from jinja2.exceptions import SecurityError
 from jinja2.sandbox import SandboxedEnvironment
 
-from .filters import datetimeformat, iso8601_to_time, regex_match, regex_replace, to_pretty_json
+from .filters import (
+    datetimeformat,
+    iso8601_to_time,
+    json_dumps,
+    regex_match,
+    regex_replace,
+    regex_search,
+    to_pretty_json,
+)
 
 
 def raise_security_exception(name):
@@ -19,3 +27,5 @@ def raise_security_exception(name):
 jinja_template_env.globals["range"] = lambda *args: raise_security_exception("range")
 jinja_template_env.filters["regex_replace"] = regex_replace
 jinja_template_env.filters["regex_match"] = regex_match
+jinja_template_env.filters["regex_search"] = regex_search
+jinja_template_env.filters["json_dumps"] = json_dumps

engine/common/tests/test_apply_jinja_template.py

@@ -14,6 +14,14 @@ def test_apply_jinja_template():
     assert payload == result
 
 
+def test_apply_jinja_template_json_dumps():
+    payload = {"name": "test"}
+
+    result = apply_jinja_template("{{ payload | json_dumps }}", payload)
+    expected = json.dumps(payload)
+    assert result == expected
+
+
 def test_apply_jinja_template_regex_match():
     payload = {"name": "test"}
 
@@ -26,6 +34,19 @@ def test_apply_jinja_template_regex_match():
         apply_jinja_template("{{ payload.name | regex_match('*') }}", payload)
 
 
+def test_apply_jinja_template_regex_search():
+    payload = {"name": "test"}
+
+    assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False"
+
+    # Check that exception is raised when regex is invalid
+    with pytest.raises(JinjaTemplateError):
+        apply_jinja_template("{{ payload.name | regex_search('*') }}", payload)
+
+
 def test_apply_jinja_template_bad_syntax_error():
     with pytest.raises(JinjaTemplateError):
         apply_jinja_template("{{%", payload={})