Merge pull request #467 from guardian/jsh/add-similarity-to-search

Add similarity ordering to rule search
guardian · Apr 8, 2024 · 57d94c7 · 57d94c7
2 parents 232e17f + bfdc202
commit 57d94c7
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 18 deletions.
diff --git a/apps/rule-manager/app/db/DbRuleDraft.scala b/apps/rule-manager/app/db/DbRuleDraft.scala
@@ -236,8 +236,29 @@ object DbRuleDraft extends SQLSyntaxSupport[DbRuleDraft] {
       tagColumn
     )
 
-    val orderByClause = if (sortBy.nonEmpty) {
-      val orderStmts = sortBy.map { sortByStr =>
+    val coalescedCols =
+      sqls"""
+        coalesce(${rd.column("pattern")}, '') ||
+        coalesce(${rd.column("description")}, '') ||
+        coalesce(${rd.column("replacement")}, '') ||
+        coalesce(${rd.column("category")}, '')
+      """
+
+    val subClauseSimilarityCol = "similarity"
+    val similarityCol = "rd.similarity"
+    val (searchClause, similaritySubClauseAlias, similarityClause) = maybeWord
+      .map { word =>
+        (
+          sqls"WHERE $coalescedCols ILIKE ${s"%$word%"}",
+          sqls", $coalescedCols <-> $word AS " + SQLSyntax.createUnsafely(subClauseSimilarityCol),
+          sqls", $similarityCol"
+        )
+      }
+      .getOrElse((sqls.empty, sqls.empty, sqls.empty))
+
+    val orderByClause = (similarityColName: String) => {
+      val maybeSimilarityOrder = maybeWord.map { _ => SQLSyntax.createUnsafely(similarityColName) }
+      val orderStmts = maybeSimilarityOrder.toList ++ sortBy.map { sortByStr =>
         val col = rd.column(sortByStr.slice(1, sortByStr.length))
         sortByStr.slice(0, 1) match {
           case "+" => sqls"$col ASC"
@@ -246,35 +267,28 @@ object DbRuleDraft extends SQLSyntaxSupport[DbRuleDraft] {
       }
 
       sqls"ORDER BY ${sqls.join(orderStmts, sqls",")}, ${rd.ruleType} ASC"
-    } else sqls.empty
-
-    val searchClause = maybeWord
-      .map { word =>
-        sqls"""WHERE
-        coalesce(${rd.column("pattern")}, '') ||
-        coalesce(${rd.column("description")}, '') ||
-        coalesce(${rd.column("replacement")}, '') ||
-        coalesce(${rd.column("category")}, '') ILIKE ${s"%$word%"}"""
-      }
-      .getOrElse(sqls.empty)
+    }
 
     val subclause = sqls"""
-          SELECT $draftRuleColumns FROM ${DbRuleDraft.as(rd)}
+          SELECT $draftRuleColumns $similaritySubClauseAlias
+            FROM ${DbRuleDraft.as(rd)}
             $searchClause
-            $orderByClause
+            ${orderByClause(subClauseSimilarityCol)}
             LIMIT $pageSize
             OFFSET ${(page - 1) * pageSize}
         """
 
     val data = sql"""
-      SELECT $dbColumns
+      SELECT $dbColumns $similarityClause
         FROM ($subclause) as rd
         LEFT JOIN ${DbRuleLive.as(rl)}
             ON ${rd.externalId} = ${rl.externalId} and ${rl.isActive} = true
         LEFT JOIN ${RuleTagDraft.as(rt)}
             ON ${rd.id} = ${rt.ruleId}
-        GROUP BY $draftRuleColumns, ${rl.externalId}, ${rl.revisionId}
-        $orderByClause
+        GROUP BY $draftRuleColumns, ${rl.externalId}, ${rl.revisionId} ${maybeWord
+        .map { _ => sqls"," + SQLSyntax.createUnsafely(similarityCol) }
+        .getOrElse(sqls.empty)}
+        ${orderByClause(similarityCol)}
     """
       .map(DbRuleDraft.fromRow)
       .list()

diff --git a/apps/rule-manager/test/db/DraftRulesSpec.scala b/apps/rule-manager/test/db/DraftRulesSpec.scala
@@ -121,6 +121,15 @@ class DraftRulesSpec extends RuleFixture with Matchers with DBTest {
     results.data shouldBe List(rule)
   }
 
+  it should "prioritise the exact match when ordering results" in { implicit session =>
+    val rule1 = insertRule(pattern = Some("Example"))
+    val rule2 = insertRule(pattern = Some("An example rule"))
+    val rule3 = insertRule(pattern = Some("Another example rule"))
+
+    val results = DbRuleDraft.searchRules(1, Some("Example"))
+    results.data.map(_.pattern) shouldBe List(rule1, rule2, rule3).map(_.pattern)
+  }
+
   it should "order rules given a sort column, ASC and DESC" in { implicit session =>
     DbRuleDraft
       .create(

diff --git a/apps/rule-manager/test/db/RuleFixture.scala b/apps/rule-manager/test/db/RuleFixture.scala
@@ -6,6 +6,8 @@ import scalikejdbc.DBSession
 import scalikejdbc.scalatest.AutoRollback
 
 trait RuleFixture extends FixtureAnyFlatSpec with AutoRollback {
+  val autoSession = AutoSession
+
   override def fixture(implicit session: DBSession) = {
     val initialRevisionId = 0
     val initialExternalId = "googleSheetId"
@@ -34,4 +36,19 @@ trait RuleFixture extends FixtureAnyFlatSpec with AutoRollback {
       .update()
       .apply()
   }
+
+  def insertRule(
+      ruleType: String = "regex",
+      pattern: Option[String] = None,
+      description: Option[String] = None
+  )(implicit session: DBSession = autoSession) = DbRuleDraft
+    .create(
+      ruleType = ruleType,
+      pattern = pattern,
+      description,
+      description,
+      user = "test.user",
+      ignore = false
+    )
+    .get
 }