Added preview and migration API endpoints for route migration from re…

…gex into jinja2 (#1715)

# What this PR does
This PR adds new API endpoints for migrating routes from regex format to
jinja2 format. The changes include the following:

* `filtering_term_as_jinja2` field to GET `channels_filters` endpoint
* A POST endpoint
`channel_filters/ABCDEF123/convert_from_regex_to_jinja2/` for migrating
routes to jinja2 format.

These new endpoints will allow users to preview and migrate their
existing regex routes to the more flexible and maintainable jinja2
format.

Check the screenshot where this endpoints will be used
<img width="407" alt="Screenshot 2023-04-14 at 09 50 23"
src="https://user-images.githubusercontent.com/2262529/231920771-20792c7e-d6ef-4681-80e1-c82bb4aa4b8e.png">

## Which issue(s) this PR fixes

## Checklist

- [ ] Unit, integration, and e2e (if applicable) tests updated
- [ ] Documentation added (or `pr:no public docs` PR label added if not
required)
- [ ] `CHANGELOG.md` updated (or `pr:no changelog` PR label added if not
required)

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added preview and migration API endpoints for route migration from regex into jinja2 ([1715](https://github.com/grafana/oncall/pull/1715))
 - Helm chart: add the option to use a helm hook for the migration job ([1386](https://github.com/grafana/oncall/pull/1386))
 - Send demo alert with dynamic payload and get demo payload example on private api ([1700](https://github.com/grafana/oncall/pull/1700))
 

engine/apps/api/serializers/channel_filter.py

@@ -26,6 +26,7 @@ class ChannelFilterSerializer(OrderedModelSerializerMixin, EagerLoadingMixin, se
         queryset=TelegramToOrganizationConnector.objects, filter_field="organization", allow_null=True, required=False
     )
     order = serializers.IntegerField(required=False)
+    filtering_term_as_jinja2 = serializers.SerializerMethodField()
 
     SELECT_RELATED = ["escalation_chain", "alert_receive_channel"]
 
@@ -45,6 +46,7 @@ class Meta:
             "notify_in_slack",
             "notify_in_telegram",
             "notification_backends",
+            "filtering_term_as_jinja2",
         ]
         read_only_fields = ["created_at", "is_default"]
         extra_kwargs = {"filtering_term": {"required": True, "allow_null": False}}
@@ -107,6 +109,16 @@ def validate_notification_backends(self, notification_backends):
             notification_backends = updated
         return notification_backends
 
+    def get_filtering_term_as_jinja2(self, obj):
+        """
+        Returns the regex filtering term as a jinja2, for the preview before migration from regex to jinja2"""
+        if obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_JINJA2:
+            return obj.filtering_term
+        elif obj.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX:
+            # Four curly braces will result in two curly braces in the final string
+            # rf"..." is a raw f string, to keep original filtering_term
+            return rf'{{{{ payload | json_dumps | regex_search("{obj.filtering_term}") }}}}'
+
 
 class ChannelFilterCreateSerializer(ChannelFilterSerializer):
     alert_receive_channel = OrganizationFilteredPrimaryKeyRelatedField(queryset=AlertReceiveChannel.objects)

engine/apps/api/tests/test_channel_filter.py

@@ -502,3 +502,67 @@ def test_channel_filter_update_invalid_notification_backends(
     assert response.status_code == status.HTTP_400_BAD_REQUEST
     assert response.json() == {"notification_backends": ["Invalid messaging backend"]}
     assert channel_filter.notification_backends is None
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "role,expected_status",
+    [
+        (LegacyAccessControlRole.ADMIN, status.HTTP_200_OK),
+        (LegacyAccessControlRole.EDITOR, status.HTTP_403_FORBIDDEN),
+        (LegacyAccessControlRole.VIEWER, status.HTTP_403_FORBIDDEN),
+    ],
+)
+def test_channel_filter_convert_from_regex_to_jinja2(
+    make_organization_and_user_with_plugin_token,
+    make_alert_receive_channel,
+    make_channel_filter,
+    make_user_auth_headers,
+    role,
+    expected_status,
+):
+    organization, user, token = make_organization_and_user_with_plugin_token(role)
+    alert_receive_channel = make_alert_receive_channel(organization)
+
+    make_channel_filter(alert_receive_channel, is_default=True)
+
+    # r"..." used to keep this string as raw string
+    regex_filtering_term = r"\".*\": \"This alert was sent by user for the demonstration purposes\""
+    final_filtering_term = r'{{ payload | json_dumps | regex_search("\".*\": \"This alert was sent by user for the demonstration purposes\"") }}'
+    payload = {"description": "This alert was sent by user for the demonstration purposes"}
+
+    regex_channel_filter = make_channel_filter(
+        alert_receive_channel,
+        filtering_term=regex_filtering_term,
+        is_default=False,
+    )
+    # Check if the filtering term is a regex
+    assert regex_channel_filter.filtering_term_type == regex_channel_filter.FILTERING_TERM_TYPE_REGEX
+    # Check if the alert is matched to the channel filter (route) regex
+    assert bool(regex_channel_filter.is_satisfying(payload)) is True
+
+    client = APIClient()
+    url = reverse("api-internal:channel_filter-detail", kwargs={"pk": regex_channel_filter.public_primary_key})
+
+    response = client.get(url, format="json", **make_user_auth_headers(user, token))
+
+    assert response.status_code == status.HTTP_200_OK
+    # Check if preview of the filtering term migration is correct
+    assert response.json()["filtering_term_as_jinja2"] == final_filtering_term
+
+    url = reverse(
+        "api-internal:channel_filter-convert-from-regex-to-jinja2",
+        kwargs={"pk": regex_channel_filter.public_primary_key},
+    )
+    response = client.post(url, **make_user_auth_headers(user, token))
+    # Only admins can convert from regex to jinja2
+    assert response.status_code == expected_status
+    if expected_status == status.HTTP_200_OK:
+        regex_channel_filter.refresh_from_db()
+        # Regex is now converted to jinja2
+        jinja2_channel_filter = regex_channel_filter
+        # Check if the filtering term is a jinja2, and if it is correct
+        assert jinja2_channel_filter.filtering_term_type == jinja2_channel_filter.FILTERING_TERM_TYPE_JINJA2
+        assert jinja2_channel_filter.filtering_term == final_filtering_term
+        # Check if the same alert is matched to the channel filter (route) new jinja2
+        assert bool(jinja2_channel_filter.is_satisfying(payload)) is True

engine/apps/api/views/channel_filter.py

@@ -41,6 +41,7 @@ class ChannelFilterView(
         "destroy": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
         "move_to_position": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
         "send_demo_alert": [RBACPermission.Permissions.INTEGRATIONS_TEST],
+        "convert_from_regex_to_jinja2": [RBACPermission.Permissions.INTEGRATIONS_WRITE],
     }
 
     model = ChannelFilter
@@ -144,3 +145,16 @@ def send_demo_alert(self, request, pk):
         except UnableToSendDemoAlert as e:
             raise BadRequest(detail=str(e))
         return Response(status=status.HTTP_200_OK)
+
+    @action(detail=True, methods=["post"])
+    def convert_from_regex_to_jinja2(self, request, pk):
+        instance = self.get_queryset().get(public_primary_key=pk)
+        if not instance.filtering_term_type == ChannelFilter.FILTERING_TERM_TYPE_REGEX:
+            raise BadRequest(detail="Only regex filtering term type is supported")
+
+        serializer_class = self.serializer_class
+
+        instance.filtering_term = serializer_class(instance).get_filtering_term_as_jinja2(instance)
+        instance.filtering_term_type = ChannelFilter.FILTERING_TERM_TYPE_JINJA2
+        instance.save()
+        return Response(status=status.HTTP_200_OK, data=serializer_class(instance).data)

engine/common/jinja_templater/filters.py

@@ -37,3 +37,17 @@ def regex_match(pattern, value):
         return bool(re.match(value, pattern))
     except (ValueError, AttributeError, TypeError):
         return None
+
+
+def regex_search(pattern, value):
+    try:
+        return bool(re.search(value, pattern))
+    except (ValueError, AttributeError, TypeError):
+        return None
+
+
+def json_dumps(value):
+    try:
+        return json.dumps(value)
+    except (ValueError, AttributeError, TypeError):
+        return None

engine/common/jinja_templater/jinja_template_env.py

@@ -3,7 +3,15 @@
 from jinja2.exceptions import SecurityError
 from jinja2.sandbox import SandboxedEnvironment
 
-from .filters import datetimeformat, iso8601_to_time, regex_match, regex_replace, to_pretty_json
+from .filters import (
+    datetimeformat,
+    iso8601_to_time,
+    json_dumps,
+    regex_match,
+    regex_replace,
+    regex_search,
+    to_pretty_json,
+)
 
 
 def raise_security_exception(name):
@@ -19,3 +27,5 @@ def raise_security_exception(name):
 jinja_template_env.globals["range"] = lambda *args: raise_security_exception("range")
 jinja_template_env.filters["regex_replace"] = regex_replace
 jinja_template_env.filters["regex_match"] = regex_match
+jinja_template_env.filters["regex_search"] = regex_search
+jinja_template_env.filters["json_dumps"] = json_dumps

engine/common/tests/test_apply_jinja_template.py

@@ -14,6 +14,14 @@ def test_apply_jinja_template():
     assert payload == result
 
 
+def test_apply_jinja_template_json_dumps():
+    payload = {"name": "test"}
+
+    result = apply_jinja_template("{{ payload | json_dumps }}", payload)
+    expected = json.dumps(payload)
+    assert result == expected
+
+
 def test_apply_jinja_template_regex_match():
     payload = {"name": "test"}
 
@@ -26,6 +34,19 @@ def test_apply_jinja_template_regex_match():
         apply_jinja_template("{{ payload.name | regex_match('*') }}", payload)
 
 
+def test_apply_jinja_template_regex_search():
+    payload = {"name": "test"}
+
+    assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True"
+    assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False"
+
+    # Check that exception is raised when regex is invalid
+    with pytest.raises(JinjaTemplateError):
+        apply_jinja_template("{{ payload.name | regex_search('*') }}", payload)
+
+
 def test_apply_jinja_template_bad_syntax_error():
     with pytest.raises(JinjaTemplateError):
         apply_jinja_template("{{%", payload={})