adding threshold and multi-line strings support

Signed-off-by: Dusan Malusev <dusan@dusanmalusev.dev>

.vscode/settings.json

@@ -12,6 +12,12 @@
       "memory": "cpp",
       "utility": "cpp",
       "iomanip": "cpp",
-      "cmath": "cpp"
+      "cmath": "cpp",
+      "array": "cpp",
+      "string_view": "cpp",
+      "initializer_list": "cpp",
+      "ranges": "cpp",
+      "span": "cpp",
+      "string": "cpp"
     }
 }

cbits.cpp

@@ -4,7 +4,6 @@
 #include <memory>
 #include <streambuf>
 
-#include <args.cc>
 #include <autotune.cc>
 #include <dictionary.cc>
 #include <fasttext.cc>
@@ -47,36 +46,45 @@ void FastText_DeleteHandle(const FastTextHandle handle)
     delete model;
 }
 
-FastText_Predict_t FastText_Predict(const FastTextHandle handle, FastText_String_t query)
+FastText_Predict_t FastText_PredictOne(const FastTextHandle handle, FastText_String_t query, float threshold)
+{
+    return FastText_Predict(handle, query, 1, threshold);
+}
+
+FastText_Predict_t FastText_Predict(const FastTextHandle handle, FastText_String_t query, int k, float threshold)
 {
     const auto model = reinterpret_cast<fasttext::FastText *>(handle);
 
     membuf sbuf(query);
     std::istream in(&sbuf);
 
     auto predictions = new std::vector<std::pair<fasttext::real, std::string>>();
-    model->predictLine(in, reinterpret_cast<std::vector<std::pair<fasttext::real, std::string>> &>(predictions), 1,
-                       0.0f);
+    model->predictLine(in, reinterpret_cast<std::vector<std::pair<fasttext::real, std::string>> &>(predictions), k,
+                       threshold);
+
+    free(query.data);
+    query.data = nullptr;
+    query.size = 0;
 
     return FastText_Predict_t{
         predictions->size(),
         (void *)predictions,
     };
 }
 
-char *FastText_Analogy(const FastTextHandle handle, const char *query, size_t length)
-{
-    return "";
+// char *FastText_Analogy(const FastTextHandle handle, const char *query, size_t length)
+// {
+//     return "";
 
-    // auto model = reinterpret_cast<fasttext::FastText *>(handle);
+//     // auto model = reinterpret_cast<fasttext::FastText *>(handle);
 
-    // model->getAnalogies(1, query, 10);
+//     // model->getAnalogies(1, query, 10);
 
-    // size_t ii = 0;
-    // auto res = json::array();
+//     // size_t ii = 0;
+//     // auto res = json::array();
 
-    // return strdup(res.dump().c_str());
-}
+//     // return strdup(res.dump().c_str());
+// }
 
 FastText_FloatVector_t FastText_Wordvec(const FastTextHandle handle, FastText_String_t word)
 {
@@ -86,6 +94,10 @@ FastText_FloatVector_t FastText_Wordvec(const FastTextHandle handle, FastText_St
     auto vec = new fasttext::Vector(dimensions);
     model->getWordVector(reinterpret_cast<fasttext::Vector &>(vec), std::string(word.data, word.size));
 
+    free(word.data);
+    word.data = nullptr;
+    word.size = 0;
+
     return FastText_FloatVector_t{
         vec->data(),
         (void *)vec,
@@ -102,6 +114,9 @@ FastText_FloatVector_t FastText_Sentencevec(const FastTextHandle handle, FastTex
 
     auto vec = new fasttext::Vector(model->getDimension());
     model->getSentenceVector(in, reinterpret_cast<fasttext::Vector &>(vec));
+    free(sentance.data);
+    sentance.data = nullptr;
+    sentance.size = 0;
 
     return FastText_FloatVector_t{
         vec->data(),

cbits.h

@@ -36,11 +36,13 @@ extern "C"
 
     FastTextHandle FastText_NewHandle(const char *path);
     void FastText_DeleteHandle(const FastTextHandle handle);
-    FastText_Predict_t FastText_Predict(const FastTextHandle handle, FastText_String_t query);
+    FastText_Predict_t FastText_Predict(const FastTextHandle handle, FastText_String_t query, int k, float threshold);
+    FastText_Predict_t FastText_PredictOne(const FastTextHandle handle, FastText_String_t query, float threshold);
+
     FastText_FloatVector_t FastText_Wordvec(const FastTextHandle handle, FastText_String_t word);
     FastText_FloatVector_t FastText_Sentencevec(const FastTextHandle handle, FastText_String_t sentance);
 
-    char *FastText_Analogy(const FastTextHandle handle, FastText_String_t query);
+    // char *FastText_Analogy(const FastTextHandle handle, FastText_String_t query);
 
     void FastText_FreeFloatVector(FastText_FloatVector_t vector);
     void FastText_FreePredict(FastText_Predict_t predict);

cmd/analogy.go

@@ -1,13 +1,7 @@
 package cmd
 
 import (
-	"fmt"
-
-	"github.com/k0kubun/pp"
 	"github.com/spf13/cobra"
-	"github.com/unknwon/com"
-
-	"github.com/nano-interactive/go-fasttext"
 )
 
 var (
@@ -20,18 +14,18 @@ var analogyCmd = &cobra.Command{
 	Short: "Perform word analogy on a query using an input model",
 	Args:  cobra.ExactArgs(1), // make sure that there is only one argument being passed in
 	Run: func(cmd *cobra.Command, args []string) {
-		if !com.IsFile(unsupervisedModelPath) {
-			fmt.Println("the file %s does not exist", unsupervisedModelPath)
-			return
-		}
+		// if !com.IsFile(unsupervisedModelPath) {
+		// 	fmt.Println("the file %s does not exist", unsupervisedModelPath)
+		// 	return
+		// }
 
-		// create a model object
-		model := fasttext.Open(unsupervisedModelPath)
-		// close the model at the end
-		defer model.Close()
-		// perform the prediction
-		analogies := model.Analogy(args[0])
-		pp.Println(analogies)
+		// // create a model object
+		// model := fasttext.Open(unsupervisedModelPath)
+		// // close the model at the end
+		// defer model.Close()
+		// // perform the prediction
+		// analogies := model.Analogy(args[0])
+		// pp.Println(analogies)
 	},
 }
 

cmd/predict.go

@@ -30,7 +30,7 @@ var predictCmd = &cobra.Command{
 		// close the model at the end
 		defer model.Close()
 		// perform the prediction
-		preds := model.Predict(args[0])
+		preds := model.Predict(args[0], 1, 0.0)
 		pp.Println(preds)
 	},
 }

fasttext.go

@@ -1,13 +1,14 @@
 package fasttext
 
-// #cgo CXXFLAGS: -I${SRCDIR}/fastText/src -I${SRCDIR} -I${SRCDIR}/include -std=c++17 -O3 -fPIC -pedantic -Wall -Wextra -Wno-sign-compare -Wno-unused-parameter
+// #cgo CXXFLAGS: -I${SRCDIR}/fastText/src -I${SRCDIR} -I${SRCDIR}/include -std=c++17 -O3 -fPIC
 // #cgo LDFLAGS: -lstdc++
 // #include <stdio.h>
 // #include <stdlib.h>
 // #include "cbits.h"
 import "C"
 
 import (
+	"strings"
 	"unsafe"
 )
 
@@ -37,17 +38,43 @@ func (handle *Model) Close() error {
 	return nil
 }
 
-// // Perform model prediction
-func (handle Model) Predict(query string) Predictions {
+func (handle Model) MultiLinePredict(query string, k int32, threshoad float32 ) []Predictions {
+	lines := strings.Split(query, "\n")
+
+	predics := make([]Predictions, 0, len(lines))
+
+	for _, line := range lines {
+		predictions := handle.Predict(line, k, threshoad)
+		predics = append(predics, predictions)
+	}
+
+	return predics
+}
+
+func (handle Model) PredictOne(query string, threshoad float32) Prediction {
 	cquery := C.CString(query)
 	defer C.free(unsafe.Pointer(cquery))
 
-	// Call the Predict function defined in cbits.cpp
-	// passing in the model handle and the query string
-	r := C.FastText_Predict(handle.p, C.FastText_String_t{
+	r := C.FastText_PredictOne(handle.p, C.FastText_String_t{
 		data: cquery,
 		size: C.size_t(len(query)),
-	})
+	}, C.float(threshoad))
+	defer C.FastText_FreePredict(r)
+
+	cPredic := C.FastText_PredictItemAt(r, C.size_t(0))
+
+	return Prediction{
+		Label:       C.GoStringN(cPredic.label.data, C.int(cPredic.label.size)),
+		Probability: float32(cPredic.probability),
+	}
+}
+
+// Perform model prediction
+func (handle Model) Predict(query string, k int32, threshoad float32) Predictions {
+	r := C.FastText_Predict(handle.p, C.FastText_String_t{
+		data: C.CString(query),
+		size: C.size_t(len(query)),
+	}, C.int(k), C.float(threshoad))
 	defer C.FastText_FreePredict(r)
 
 	predictions := make(Predictions, r.size)
@@ -82,11 +109,8 @@ func (handle Model) Predict(query string) Predictions {
 // }
 
 func (handle Model) Wordvec(word string) []float32 {
-	cquery := C.CString(word)
-	defer C.free(unsafe.Pointer(cquery))
-
 	r := C.FastText_Wordvec(handle.p, C.FastText_String_t{
-		data: cquery,
+		data: C.CString(word),
 		size: C.size_t(len(word)),
 	})
 	defer C.FastText_FreeFloatVector(r)
@@ -99,11 +123,8 @@ func (handle Model) Wordvec(word string) []float32 {
 
 // Requires sentence ends with </s>
 func (handle Model) Sentencevec(query string) []float32 {
-	cquery := C.CString(query)
-	defer C.free(unsafe.Pointer(cquery))
-
 	r := C.FastText_Sentencevec(handle.p, C.FastText_String_t{
-		data: cquery,
+		data: C.CString(query),
 		size: C.size_t(len(query)),
 	})