in-memory postings with dummy tokenization

src/index.rs

@@ -1,10 +1,10 @@
-use std::fs;
+use std::{collections::HashMap, fs};
 
 use crate::trie::Trie;
 
 pub struct Index {
     vocab: Trie,
-    postings: Vec<Vec<i32>>,
+    postings: Vec<Vec<usize>>,
 }
 
 impl Index {
@@ -13,20 +13,33 @@ impl Index {
         Index { vocab, postings }
     }
 
-    fn build_internal(directory: &str) -> (Trie, Vec<Vec<i32>>) {
+    fn build_internal(directory: &str) -> (Trie, Vec<Vec<usize>>) {
         let paths = fs::read_dir(directory).unwrap();
 
-        let mut vocab = Trie::new();
+        let mut seen: HashMap<String, usize> = HashMap::new();
+        let mut vocab: Trie = Trie::new();
         let mut postings = Vec::new();
 
-        let mut i = 0;
-
-        for path in paths {
-            let s = fs::read_to_string(path.unwrap().path()).unwrap();
+        for (doc_idx, path) in paths.enumerate() {
+            // replace all punctuation in the string
+            let s = fs::read_to_string(path.unwrap().path())
+                .unwrap()
+                .replace(&['(', ')', ',', '\"', '.', ';', ':', '\''][..], " ");
 
+            // split and insert into the vocabulary if needed
             for w in s.split_ascii_whitespace() {
-                vocab.insert(w, i);
-                i += 1;
+                let lowercase_word = w.to_lowercase();
+                let to_insert = lowercase_word.clone();
+
+                if !seen.contains_key(to_insert.as_str()) {
+                    println!("New word {to_insert}");
+                    let word_idx = seen.len();
+                    vocab.insert(to_insert.as_str(), word_idx);
+                    seen.insert(to_insert, word_idx);
+                    postings.push(Vec::new())
+                }
+
+                postings[*seen.get(lowercase_word.as_str()).unwrap()].push(doc_idx);
             }
         }
 
@@ -41,8 +54,20 @@ mod tests {
 
     #[test]
     fn test_build() {
-        let i = Index::new("test_data");
+        let i = Index::new("test_data/index");
+
+        assert!(i.vocab.get("apple").is_some());
+        assert!(i.vocab.get("park").is_some());
+
+        let apple_idx = i.vocab.get("apple").unwrap();
+        let binary_idx = i.vocab.get("binary").unwrap();
+
+        let apple_postings = i.postings.get(apple_idx).unwrap();
+        let binary_postings = i.postings.get(binary_idx).unwrap();
+
+        assert!(apple_postings.len() == 2);
+        assert!(binary_postings.len() == 1);
 
-        assert!(i.vocab.get("In").is_some())
+        assert!(!apple_postings.contains(binary_postings.first().unwrap()));
     }
 }

src/trie.rs

@@ -2,7 +2,7 @@ use std::collections::{HashMap, VecDeque};
 
 #[derive(Default)]
 struct Node {
-    value: Option<i32>,
+    value: Option<usize>,
     children: HashMap<char, Node>,
 }
 
@@ -17,7 +17,7 @@ impl Trie {
         }
     }
 
-    pub fn insert(&mut self, word: &str, value: i32) {
+    pub fn insert(&mut self, word: &str, value: usize) {
         let mut node = &mut self.root;
 
         for c in word.chars() {
@@ -27,11 +27,11 @@ impl Trie {
         node.value = Some(value);
     }
 
-    pub fn get(&self, word: &str) -> Option<i32> {
+    pub fn get(&self, word: &str) -> Option<usize> {
         self.get_internal(word).and_then(|n| n.value)
     }
 
-    pub fn get_by_prefix(&self, prefix: &str) -> Vec<i32> {
+    pub fn get_by_prefix(&self, prefix: &str) -> Vec<usize> {
         self.get_internal(prefix)
             .map_or_else(Vec::new, |n| self.visit(n))
     }
@@ -49,8 +49,8 @@ impl Trie {
         Some(node)
     }
 
-    fn visit(&self, node: &Node) -> Vec<i32> {
-        let mut res: Vec<i32> = Vec::new();
+    fn visit(&self, node: &Node) -> Vec<usize> {
+        let mut res: Vec<usize> = Vec::new();
         let mut queue: VecDeque<&Node> = VecDeque::new();
         queue.push_back(node);
 

test_data/index/1

@@ -0,0 +1 @@
+The apple fell from the tree branch.

test_data/index/2

@@ -0,0 +1 @@
+A binary tree is a sorted data structure.

test_data/index/3

@@ -0,0 +1 @@
+Apple Park is filled with trees.