From 0014ef4fb415999a69f1f7aa3c7e5d0d49b07456 Mon Sep 17 00:00:00 2001
From: IanCa <ianrcallanan@gmail.com>
Date: Thu, 5 Oct 2023 16:27:07 -0500
Subject: [PATCH] Update basic_search to support words taht must not appear

---
 hed/models/basic_search.py        |  88 ++++++-----
 tests/models/test_basic_search.py | 243 +++++++++++++++++-------------
 2 files changed, 189 insertions(+), 142 deletions(-)

diff --git a/hed/models/basic_search.py b/hed/models/basic_search.py
index ae47b71e..9301a0cc 100644
--- a/hed/models/basic_search.py
+++ b/hed/models/basic_search.py
@@ -8,14 +8,18 @@ def find_matching(series, search_string, regex=False):
     """ Finds lines in the series that match the search string and returns a mask.
 
     Syntax Rules:
-        - '@': Prefixing a term in the search string means the object must appear anywhere within a line.
+        - '@': Prefixing a term in the search string means the term must appear anywhere within a line.
+        - '~': Prefixing a term in the search string means the term must NOT appear within a line.
         - Parentheses: Elements within parentheses must appear in the line with the same level of nesting.
-                eg: Search string: "(A), (B)" will match "(A), (B, C)", but not "(A, B)", since they don't
-                    start in the same group.
+                e.g.: Search string: "(A), (B)" will match "(A), (B, C)", but not "(A, B)", since they don't
+                      start in the same group.
         - "LongFormTag*": A * will match any remaining word(anything but a comma or parenthesis)
         - An individual term can be arbitrary regex, but it is limited to single continuous words.
 
     Notes:
+        - Specific words only care about their level relative to other specific words, not overall.
+              e.g. "(A, B)" will find: "A, B", "(A, B)", (A, (C), B)", or ((A, B))"
+        - If you have no grouping or anywhere words in the search, it assumes all terms are anywhere words.
         - The format of the series should match the format of the search string, whether it's in short or long form.
         - To enable support for matching parent tags, ensure that both the series and search string are in long form.
 
@@ -33,19 +37,19 @@ def find_matching(series, search_string, regex=False):
     if not regex:
         # Replace *'s with a reasonable value for people who don't know regex
         search_string = re.sub(r'(?<!\.)\*', '.*?', search_string)
-    anywhere_words, specific_words = find_words(search_string)
+    anywhere_words, negative_words, specific_words = find_words(search_string)
+    # If we have no nesting or anywhere words, assume they don't care about level
+    if "(" not in search_string and "@" not in search_string:
+        anywhere_words += specific_words
+        specific_words = []
     delimiter_map = construct_delimiter_map(search_string, specific_words)
-    source_words = anywhere_words + specific_words
+    candidate_indexes = _verify_basic_words(series, anywhere_words, negative_words)
 
-    # Create a set of series of masks to determine which rows contain each individual word
-    candidate_indexes = set(series.index)
-
-    # Loop through source_words to filter down candidate_indexes
-    for word in source_words:
+    # do a basic check for all specific words(this doesn't verify word delimiters)
+    for word in specific_words:
         matches = series.str.contains(word, regex=True)
         current_word_indexes = set(matches[matches].index.tolist())
 
-        # Update candidate_indexes by taking the intersection with current_word_indexes
         candidate_indexes &= current_word_indexes
 
         if not candidate_indexes:
@@ -53,40 +57,63 @@ def find_matching(series, search_string, regex=False):
 
     candidate_indexes = sorted(candidate_indexes)
 
-    full_mask = pd.Series(False, index=series.index)
-
-    candidate_series = series[candidate_indexes]
+    full_mask = pd.Series(False, index=series.index, dtype=bool)
 
-    mask = candidate_series.apply(verify_search_delimiters, args=(anywhere_words, specific_words, delimiter_map))
-    full_mask.loc[candidate_indexes] = mask
+    if candidate_indexes:
+        if specific_words:
+            candidate_series = series[candidate_indexes]
+            mask = candidate_series.apply(verify_search_delimiters, args=(specific_words, delimiter_map))
+            full_mask.loc[candidate_indexes] = mask
+        else:
+            full_mask.loc[candidate_indexes] = True
 
     return full_mask
 
 
+def _get_word_indexes(series, word):
+    pattern = r'(?:[ ,()]|^)' + word + r'(?:[ ,()]|$)'
+    matches = series.str.contains(pattern, regex=True)
+    return set(matches[matches].index.tolist())
+
+
+def _verify_basic_words(series, anywhere_words, negative_words):
+    candidate_indexes = set(series.index)
+    for word in anywhere_words:
+        current_word_indexes = _get_word_indexes(series, word)
+        candidate_indexes &= current_word_indexes
+
+    for word in negative_words:
+        current_word_indexes = _get_word_indexes(series, word)
+        candidate_indexes -= current_word_indexes
+    return candidate_indexes
+
+
 def find_words(search_string):
-    """ Extract all words in the search string.  Dividing them into words that must be relative to each other,
-         and words that can be anywhere.
+    """
+    Extract words in the search string based on their prefixes.
 
     Args:
         search_string (str): The search query string to parse.
-                             Words prefixed with '@' are 'anywhere' words.
+                             Words can be prefixed with '@' or '~'.
 
     Returns:
-        tuple: A tuple containing two lists:
-            - anywhere_words (list of str): Words that can appear anywhere in the text.
-            - specific_words (list of str): Words that must appear relative to other terms.
+        list: A list containing three lists:
+            - Words prefixed with '@'
+            - Words prefixed with '~'
+            - Words with no prefix
     """
-    # Match sequences of characters that are not commas, parentheses, or standalone spaces.
+    # Match sequences of characters that are not commas or parentheses.
     pattern = r'[^,()]+'
     words = re.findall(pattern, search_string)
 
     # Remove any extraneous whitespace from each word
     words = [word.strip() for word in words if word.strip()]
 
-    anywhere_words = [word[1:] for word in words if word.startswith("@")]
-    specific_words = [word for word in words if not word.startswith("@")]
+    at_words = [word[1:] for word in words if word.startswith("@")]
+    tilde_words = [word[1:] for word in words if word.startswith("~")]
+    no_prefix_words = [word for word in words if not word.startswith("~") and not word.startswith("@")]
 
-    return anywhere_words, specific_words
+    return [at_words, tilde_words, no_prefix_words]
 
 
 def check_parentheses(text):
@@ -180,12 +207,11 @@ def construct_delimiter_map(text, words):
     return delimiter_map
 
 
-def verify_search_delimiters(text, anywhere_words, specific_words, delimiter_map):
+def verify_search_delimiters(text, specific_words, delimiter_map):
     """ Verifies if the text contains specific words with expected delimiters between them.
 
     Args:
         text (str): The text to search in.
-        anywhere_words (list of str): Words that can appear anywhere in the text.
         specific_words (list of str): Words that must appear relative to other words in the text
         delimiter_map (dict): A dictionary specifying expected delimiters between pairs of specific words.
 
@@ -194,12 +220,6 @@ def verify_search_delimiters(text, anywhere_words, specific_words, delimiter_map
     """
     locations = defaultdict(list)
 
-    # Check for anywhere words
-    for word in anywhere_words:
-        pattern = r'(?:[ ,()]|^)(' + word + r')(?:[ ,()]|$)'
-        if not any(re.finditer(pattern, text)):
-            return False
-
     # Find all locations for each word in the text
     for word in specific_words:
         for match in re.finditer(r'(?:[ ,()]|^)(' + word + r')(?:[ ,()]|$)', text):
diff --git a/tests/models/test_basic_search.py b/tests/models/test_basic_search.py
index 0a942b93..36fcc168 100644
--- a/tests/models/test_basic_search.py
+++ b/tests/models/test_basic_search.py
@@ -35,37 +35,37 @@ def test_find_matching_results(self):
 class TestFindWords(unittest.TestCase):
     def test_basic(self):
         search_string = "@global (local1, local2)"
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, ['global'])
         self.assertEqual(specific_words, ['local1', 'local2'])
 
     def test_no_anywhere_words(self):
         search_string = "(local1, local2)"
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, [])
         self.assertEqual(specific_words, ['local1', 'local2'])
 
     def test_no_specific_words(self):
         search_string = "@global1, @global2"
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, ['global1', 'global2'])
         self.assertEqual(specific_words, [])
 
     def test_empty_string(self):
         search_string = ""
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, [])
         self.assertEqual(specific_words, [])
 
     def test_mixed_words(self):
         search_string = "@global (local1, local2), @another_global"
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, ['global', 'another_global'])
         self.assertEqual(specific_words, ['local1', 'local2'])
 
     def test_whitespace(self):
         search_string = " @Global ,  ( local1 , local2 ) "
-        anywhere_words, specific_words = find_words(search_string)
+        anywhere_words, _, specific_words = find_words(search_string)
         self.assertEqual(anywhere_words, ['Global'])
         self.assertEqual(specific_words, ['local1', 'local2'])
 
@@ -138,118 +138,30 @@ def test_multiple_words(self):
 
 
 class TestVerifyDelimiters(unittest.TestCase):
-    def base_verify_func(self, query_text, text, anywhere_words, specific_words, expected_result):
+    def base_verify_func(self, query_text, text, specific_words, expected_result):
         delimiter_map = construct_delimiter_map(query_text, specific_words)
-        actual_result = verify_search_delimiters(text, anywhere_words, specific_words, delimiter_map)
+        actual_result = verify_search_delimiters(text, specific_words, delimiter_map)
         self.assertEqual(actual_result, expected_result)
 
     def test_all_conditions_met(self):
         query_text = "word0,((word1),word2)"
         specific_words = ["word0", "word1", "word2"]
         text = "word0,((word1),word2)"
-        self.base_verify_func(query_text, text, [], specific_words, True)
+        self.base_verify_func(query_text, text, specific_words, True)
         text = "((word1),word2), word0"
-        self.base_verify_func(query_text, text, [], specific_words, True)
+        self.base_verify_func(query_text, text, specific_words, True)
         text = "word0,(word2, (word1))"
-        self.base_verify_func(query_text, text, [], specific_words, True)
+        self.base_verify_func(query_text, text, specific_words, True)
         text = "word0,((word1),(ExtraGroup),word2)"
-        self.base_verify_func(query_text, text, [], specific_words, True)
+        self.base_verify_func(query_text, text, specific_words, True)
         text = "word0,((word2),word1)"
-        self.base_verify_func(query_text, text, [], specific_words, False)
+        self.base_verify_func(query_text, text, specific_words, False)
         text = "((word1),word0), word2"
-        self.base_verify_func(query_text, text, [], specific_words, False)
+        self.base_verify_func(query_text, text, specific_words, False)
         text = "word0,((word1))"
-        self.base_verify_func(query_text, text, [], specific_words, False)
+        self.base_verify_func(query_text, text, specific_words, False)
         text = "(word1),(ExtraGroup),word2)"
-        self.base_verify_func(query_text, text, [], specific_words, False)
-
-    def test_complex_case_with_word_identifiers(self):
-        query_text = "word0,((word1),@word2,@word3,word4)"
-        specific_words = ["word0", "word1", "word4"]
-        anywhere_words = ["word2", "word3"]
-        text = "word0,((word1),word2,word3,word4)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-        text = "word2,word0,((word1),word3,word4)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-        text = "word3,((word1),word2,word4),word0"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-        text = "word0,((word1),word4),word2,word3"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-        text = "word0,word1,word4,word2"  # Incorrect delimiters
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-        text = "word2,word3"  # Missing specific words
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-    def test_very_complex_case_with_word_identifiers(self):
-        query_text = "word0,(((word1,word2),@word3)),((word4,word5)))"
-        specific_words = ["word0", "word1", "word2", "word4", "word5"]
-        anywhere_words = ["word3"]
-
-        # Test case where all conditions are met
-        text = "word0,(((word1,word2),word3)),((word4,word5)))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Test case with anywhere words out of specific context but still in the string
-        text = "word3,word0,(((word1,word2))),((word4,word5)))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Test case with correct specific words but incorrect delimiters
-        text = "word0,((word1,word2),word3),(word4,word5)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-        # Test case missing one specific word
-        text = "word0,(((word1,word2),word3)),(word4))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-        # Test case missing anywhere word
-        text = "word0,(((word1,word2))),((word4,word5)))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-    def test_incorrect_single_delimiter(self):
-        query_text = "word0,((word1)),word2"
-        specific_words = ["word0", "word1", "word2"]
-        anywhere_words = []
-
-        # Positive case 1: Exact match
-        text = "word0,((word1)),word2"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Positive case 2: Additional parentheses around the entire sequence
-        text = "(word0,((word1)),word2)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Single closing parenthesis missing between word1 and word2
-        text = "word0,((word1),word2)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-        # Single opening parenthesis missing between word0 and word1
-        text = "word0,(word1)),word2"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-    def test_mismatched_parentheses(self):
-        query_text = "word0,((word1)),(word2,word3)"
-        specific_words = ["word0", "word1", "word2", "word3"]
-        anywhere_words = []
-
-        # Positive case 1: Exact match
-        text = "word0,((word1)),(word2,word3)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Positive case 2: Reordered sequence with the same delimiters
-        text = "(word2,word3),word0,((word1))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Positive case 3: Additional text in between but the delimiters remain the same
-        text = "word0,someExtraText,((word1)),someMoreText,(word2,word3)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, True)
-
-        # Extra closing parenthesis between word2 and word3
-        text = "word0,((word1),(word2,word3))"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
-
-        # Extra opening parenthesis between word1 and word2
-        text = "word0,((word1),((word2,word3)"
-        self.base_verify_func(query_text, text, anywhere_words, specific_words, False)
+        self.base_verify_func(query_text, text, specific_words, False)
 
     def test_wildcard_matching_verify_delimiters(self):
         query_text = "word0, ((word1.*?)), word2.*?"
@@ -257,14 +169,15 @@ def test_wildcard_matching_verify_delimiters(self):
 
         # Positive test cases
         text = "((word1)), word0, word2X"
-        self.assertTrue(verify_search_delimiters(text, [], ["word0", "word1.*?", "word2.*?"], delimiter_map))
+        self.assertTrue(verify_search_delimiters(text, ["word0", "word1.*?", "word2.*?"], delimiter_map))
 
         text = "word0, ((word1Y)), word2Z"
-        self.assertTrue(verify_search_delimiters(text, [], ["word0", "word1.*?", "word2.*?"], delimiter_map))
+        self.assertTrue(verify_search_delimiters(text, ["word0", "word1.*?", "word2.*?"], delimiter_map))
 
         # Negative test cases
         text = "word0, (word1), word2"
-        self.assertFalse(verify_search_delimiters(text, [], ["word0", "word1.*?", "word2.*?"], delimiter_map))
+        self.assertFalse(verify_search_delimiters(text, ["word0", "word1.*?", "word2.*?"], delimiter_map))
+
 
 class TestFindMatching(unittest.TestCase):
     def base_find_matching(self, series, search_string, expected):
@@ -272,6 +185,18 @@ def base_find_matching(self, series, search_string, expected):
         self.assertTrue(all(mask == expected), f"Expected {expected}, got {mask}")
 
     def test_basic_matching(self):
+        series = pd.Series([
+            "word0, word1, word2",
+            "word0, (word1, word2)"
+        ])
+        search_string = "word0, word1"
+        expected = pd.Series([True, True])
+        self.base_find_matching(series, search_string, expected)
+        search_string = "(word0, word1)"
+        expected = pd.Series([True, False])
+        self.base_find_matching(series, search_string, expected)
+
+    def test_group_matching(self):
         series = pd.Series([
             "(word1), word0, ((word2))",
             "word0, ((word1)), word2",
@@ -306,8 +231,110 @@ def test_wildcard_matching(self):
         series = pd.Series([
             "word2, word0, ((word1X))",
             "word0, ((word1Y)), word2Z",
+            "word0, ((word1)), word2",
             "word0, (word1), word2"
         ])
         search_string = "word0, ((word1*)), word2*"
-        expected = pd.Series([True, True, False])
+        expected = pd.Series([True, True, True, False])
+        self.base_find_matching(series, search_string, expected)
+
+    def test_complex_case_with_word_identifiers(self):
+        query_text = "word0, ((word1), @word2, @word3, word4)"
+        series = pd.Series([
+            "word0, ((word1), word2, word3, word4)",
+            "word2, word0, ((word1), word3, word4)",
+            "word3, ((word1), word2, word4), word0",
+            "word0, ((word1), word4), word2, word3",
+            "word0, word1, word4, word2",
+            "word2, word3"
+        ])
+        expected = pd.Series([True, True, True, True, False, False])
+
+        self.base_find_matching(series, query_text, expected)
+
+    def test_very_complex_case_with_word_identifiers(self):
+        query_text = "word0, (((word1, word2), @word3)), ((word4, word5)))"
+        series = pd.Series([
+            "word0, (((word1, word2), word3)), ((word4, word5)))",
+            "word3, word0, (((word1, word2))), ((word4, word5)))",
+            "word0, ((word1, word2), word3), (word4, word5)",
+            "word0, (((word1, word2), word3)), (word4)",
+            "word0, (((word1, word2))), ((word4, word5)))"
+        ])
+        expected = pd.Series([True, True, False, False, False])
+
+        self.base_find_matching(series, query_text, expected)
+
+    def test_incorrect_single_delimiter(self):
+        query_text = "word0, ((word1)), word2"
+        series = pd.Series([
+            "word0, ((word1)), word2",
+            "(word0, ((word1)), word2)",
+            "word0, ((word1), word2)",
+            "word0, (word1)), word2"
+        ])
+        expected = pd.Series([True, True, False, False])
+        self.base_find_matching(series, query_text, expected)
+
+    def test_mismatched_parentheses2(self):
+        query_text = "word0, ((word1)), (word2, word3)"
+        series = pd.Series([
+            "word0, ((word1)), (word2, word3)",
+            "(word2, word3), word0, ((word1))",
+            "word0, someExtraText, ((word1)), someMoreText, (word2, word3)",
+            "word0, ((word1), (word2, word3))",
+            "word0, ((word1), ((word2, word3)"
+        ])
+        expected = pd.Series([True, True, True, False, False])
+        self.base_find_matching(series, query_text, expected)
+
+    def test_negative_words(self):
+        series = pd.Series([
+            "word0, word1",
+            "word0, word2",
+            "word0, word2, word3",
+            "word0, (word1), word2",
+            "word0, (word2, word3), word1",
+            "word0, word1suffix",
+        ])
+
+        # 1. Basic Negative Test Case
+        search_string1 = "~word1, word0"
+        expected1 = pd.Series([False, True, True, False, False, True])
+
+        # 2. Two Negative Words
+        search_string2 = "~word1, ~word3, word0"
+        expected2 = pd.Series([False, True, False, False, False, True])
+
+        # 3. Combination of Negative and Mandatory Words
+        search_string3 = "@word2, ~word1, word0"
+        expected3 = pd.Series([False, True, True, False, False, False])
+
+        # 4. Negative Words with Wildcards
+        search_string4 = "word0, ~word1*"
+        expected4 = pd.Series([False, True, True, False, False, False])
+
+        # Running tests
+        self.base_find_matching(series, search_string1, expected1)
+        self.base_find_matching(series, search_string2, expected2)
+        self.base_find_matching(series, search_string3, expected3)
+        self.base_find_matching(series, search_string4, expected4)
+
+    def test_negative_words_group(self):
+        series = pd.Series([
+            "word0, (word1, (word2))",
+            "word0, (word1, (word2)), word3",
+            "word0, (word1, (word2), word3)",
+            "word0, (word1, (word2, word3))",
+            ])
+        search_string = "word0, (word1, (word2))"
+        expected = pd.Series([True, True, True, True])
+        self.base_find_matching(series, search_string, expected)
+
+        search_string = "word0, (word1, (word2)), ~word3"
+        expected = pd.Series([True, False, False, False])
+        self.base_find_matching(series, search_string, expected)
+
+        search_string = "word0, (word1, (word2), ~word3)"
+        expected = pd.Series([True, False, False, False])
         self.base_find_matching(series, search_string, expected)