Merge pull request #140 from snipsco/release/0.64.3

Release 0.64.3

CHANGELOG.md

@@ -1,6 +1,17 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.64.3] - 2019-04-29
+### Fixed
+- Make the `WrongModelVersion` error message intelligible [#133](https://github.com/snipsco/snips-nlu-rs/pull/133)
+- Fix error handling in Python wrapper [#134](https://github.com/snipsco/snips-nlu-rs/pull/134)
+- Return an error when using unknown intents in whitelist or blacklist [#136](https://github.com/snipsco/snips-nlu-rs/pull/136)
+- Fix issue with stop words in `DeterministicIntentParser` [#137](https://github.com/snipsco/snips-nlu-rs/pull/137)
+- Fix caching issue in `CustomEntityParser` [#138](https://github.com/snipsco/snips-nlu-rs/pull/138)
+
+### Changed
+- Re-score ambiguous `DeterministicIntentParser` results based on slots [#139](https://github.com/snipsco/snips-nlu-rs/pull/139)
+
 ## [0.64.2] - 2019-04-09
 ### Fixed
 - Fix handling of ambiguous utterances in `DeterministicIntentParser` [#129](https://github.com/snipsco/snips-nlu-rs/pull/129)
@@ -189,6 +200,7 @@ being statically hardcoded, reducing the binary size by 31Mb.
 - Improve support for japanese
 - Rename python package to `snips_nlu_rust`
 
+[0.64.3]: https://github.com/snipsco/snips-nlu-rs/compare/0.64.2...0.64.3
 [0.64.2]: https://github.com/snipsco/snips-nlu-rs/compare/0.64.1...0.64.2
 [0.64.1]: https://github.com/snipsco/snips-nlu-rs/compare/0.64.0...0.64.1
 [0.64.0]: https://github.com/snipsco/snips-nlu-rs/compare/0.63.1...0.64.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-lib"
-version = "0.64.2"
+version = "0.64.3"
 authors = [
     "Adrien Ball <adrien.ball@snips.ai>",
     "Clement Doumouro <clement.doumouro@snips.ai>",

README.rst

@@ -39,7 +39,7 @@ Properly trained, the Snips NLU engine will be able to extract structured data s
    {
       "intent": {
          "intentName": "searchWeatherForecast",
-         "probability": 0.95
+         "confidenceScore": 0.95
       },
       "slots": [
          {
@@ -67,7 +67,7 @@ the `Snips NLU python library <https://github.com/snipsco/snips-nlu>`_.
 Example and API Usage
 ---------------------
 
-The `interactive parsing CLI <examples/interactive_parsing_cli>`_  is a good example
+The `interactive parsing CLI <examples/interactive_parsing_cli.rs>`_  is a good example
 of to how to use ``snips-nlu-rs``.
 
 Here is how you can run the CLI example:

ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-ffi"
-version = "0.64.2"
+version = "0.64.3"
 edition = "2018"
 authors = [
     "Adrien Ball <adrien.ball@snips.ai>",

ffi/cbindgen.toml

@@ -2,7 +2,7 @@ language = "c"
 
 include_guard = "LIBSNIPS_NLU_H_"
 
-header = "#define SNIPS_NLU_VERSION \"0.64.2\""
+header = "#define SNIPS_NLU_VERSION \"0.64.3\""
 
 [parse]
 parse_deps = true

platforms/c/libsnips_nlu.h

@@ -1,4 +1,4 @@
-#define SNIPS_NLU_VERSION "0.64.2"
+#define SNIPS_NLU_VERSION "0.64.3"
 
 #ifndef LIBSNIPS_NLU_H_
 #define LIBSNIPS_NLU_H_

platforms/kotlin/build.gradle

@@ -11,7 +11,7 @@ buildscript {
 
 apply plugin: 'kotlin'
 
-version = "0.64.2"
+version = "0.64.3"
 group = "ai.snips"
 
 repositories {

platforms/python/ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-python-ffi"
-version = "0.64.2"
+version = "0.64.3"
 authors = ["Adrien Ball <adrien.ball@snips.ai>"]
 edition = "2018"
 
@@ -11,4 +11,4 @@ crate-type = ["cdylib"]
 [dependencies]
 libc = "0.2"
 ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "4292ad9" }
-snips-nlu-ffi = { git = "https://github.com/snipsco/snips-nlu-rs", tag = "0.64.2" }
+snips-nlu-ffi = { git = "https://github.com/snipsco/snips-nlu-rs", tag = "0.64.3" }

platforms/python/snips_nlu_rust/__version__

@@ -1 +1 @@
-0.64.2
+0.64.3

platforms/python/snips_nlu_rust/nlu_engine.py

@@ -4,10 +4,11 @@
 
 import json
 from builtins import object, str
-from ctypes import byref, c_char_p, c_void_p, pointer, string_at, c_char, c_int
+from ctypes import byref, c_char_p, c_void_p, string_at, c_char, c_int
 from pathlib import Path
 
-from snips_nlu_rust.utils import lib, string_pointer, CStringArray
+from snips_nlu_rust.utils import (
+    lib, string_pointer, CStringArray, check_ffi_error)
 
 
 class NLUEngine(object):
@@ -32,69 +33,73 @@ class NLUEngine(object):
         ...     engine_dir="/path/to/nlu_engine")
         >>> inference_engine.parse("Turn on the lights in the kitchen")
     """
+
     def __init__(self, engine_dir=None, engine_bytes=None):
-        exit_code = 1
         self._engine = None
 
         if engine_dir is None and engine_bytes is None:
             raise ValueError("Please specify engine_dir or engine_bytes")
 
         if engine_dir is not None:
             engine_dir = Path(engine_dir)
-            if not engine_dir.is_dir():
-                raise OSError("NLU engine directory not found: %s"
-                              % str(engine_dir))
-            self._engine = pointer(c_void_p())
+            self._engine = c_void_p()
             exit_code = lib.ffi_snips_nlu_engine_create_from_dir(
                 str(engine_dir).encode("utf8"), byref(self._engine))
-        elif engine_bytes is not None:
-            self._engine = pointer(c_void_p())
+            err_msg = "Something went wrong when creating the engine from " \
+                      "directory"
+
+        else:
+            self._engine = c_void_p()
             bytearray_type = c_char * len(engine_bytes)
             exit_code = lib.ffi_snips_nlu_engine_create_from_zip(
                 bytearray_type.from_buffer(engine_bytes), len(engine_bytes),
                 byref(self._engine))
+            err_msg = "Something went wrong when creating the engine from " \
+                      "bytes"
 
-        if exit_code:
-            raise ImportError('Something wrong happened while creating the '
-                              'intent parser. See stderr.')
+        check_ffi_error(exit_code, err_msg)
 
     def parse(self, query, intents_whitelist=None, intents_blacklist=None):
         """Extracts intent and slots from an input query
 
         Args:
             query (str): input to process
-            intents_whitelist (list of str, optional): if defined, this will restrict the scope of
-                intent parsing to the provided intents
-            intents_blacklist (list of str, optional): if defined, these intents will be excluded
-                from the scope of intent parsing
+            intents_whitelist (list of str, optional): if defined, this will
+                restrict the scope of intent parsing to the provided intents
+            intents_blacklist (list of str, optional): if defined, these
+                intents will be excluded from the scope of intent parsing
 
         Returns:
             A python dict containing data about intent and slots. See
-            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing for details about the
-            format.
+            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing
+            for details about the format.
         """
         if intents_whitelist is not None:
-            if not all(isinstance(intent, str) for intent in intents_whitelist):
-                raise TypeError(
-                    "Expected 'intents_whitelist' to contain objects of type 'str'")
+            if not all(
+                    isinstance(intent, str) for intent in intents_whitelist):
+                raise TypeError("Expected 'intents_whitelist' to contain "
+                                "objects of type 'str'")
             intents = [intent.encode("utf8") for intent in intents_whitelist]
             arr = CStringArray()
             arr.size = c_int(len(intents))
             arr.data = (c_char_p * len(intents))(*intents)
             intents_whitelist = byref(arr)
         if intents_blacklist is not None:
-            if not all(isinstance(intent, str) for intent in intents_blacklist):
-                raise TypeError(
-                    "Expected 'intents_blacklist' to contain objects of type 'str'")
+            if not all(
+                    isinstance(intent, str) for intent in intents_blacklist):
+                raise TypeError("Expected 'intents_blacklist' to contain "
+                                "objects of type 'str'")
             intents = [intent.encode("utf8") for intent in intents_blacklist]
             arr = CStringArray()
             arr.size = c_int(len(intents))
             arr.data = (c_char_p * len(intents))(*intents)
             intents_blacklist = byref(arr)
         with string_pointer(c_char_p()) as ptr:
-            lib.ffi_snips_nlu_engine_run_parse_into_json(
-                self._engine, query.encode("utf8"), intents_whitelist, intents_blacklist,
-                byref(ptr))
+            exit_code = lib.ffi_snips_nlu_engine_run_parse_into_json(
+                self._engine, query.encode("utf8"), intents_whitelist,
+                intents_blacklist, byref(ptr))
+            msg = "Something went wrong when parsing query '%s'" % query
+            check_ffi_error(exit_code, msg)
             result = string_at(ptr)
 
         return json.loads(result.decode("utf8"))
@@ -108,13 +113,18 @@ def get_slots(self, query, intent):
 
         Returns:
             A list of slots. See
-            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing for details about the
-            format.
+            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing
+            for details about the format.
         """
         with string_pointer(c_char_p()) as ptr:
-            lib.ffi_snips_nlu_engine_run_get_slots_into_json(
-                self._engine, query.encode("utf8"), intent.encode("utf8"), byref(ptr))
+            exit_code = lib.ffi_snips_nlu_engine_run_get_slots_into_json(
+                self._engine, query.encode("utf8"), intent.encode("utf8"),
+                byref(ptr))
+            msg = "Something went wrong when extracting slots from query " \
+                  "'%s' with intent '%s'" % (query, intent)
+            check_ffi_error(exit_code, msg)
             result = string_at(ptr)
+
         return json.loads(result.decode("utf8"))
 
     def get_intents(self, query):
@@ -125,12 +135,15 @@ def get_intents(self, query):
 
         Returns:
             A list of intents along with their probability. See
-            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing for details about the
-            format.
+            https://snips-nlu.readthedocs.io/en/latest/tutorial.html#parsing
+            for details about the format.
         """
         with string_pointer(c_char_p()) as ptr:
-            lib.ffi_snips_nlu_engine_run_get_intents_into_json(
+            exit_code = lib.ffi_snips_nlu_engine_run_get_intents_into_json(
                 self._engine, query.encode("utf8"), byref(ptr))
+            msg = "Something went wrong when extracting intents from query " \
+                  "'%s'" % query
+            check_ffi_error(exit_code, msg)
             result = string_at(ptr)
         return json.loads(result.decode("utf8"))
 

platforms/python/snips_nlu_rust/tests/test_nlu_engine_wrapper.py

@@ -19,6 +19,12 @@ def test_should_load_from_dir_and_parse(self):
         # Then
         self.assertEqual("MakeCoffee", res["intent"]["intentName"])
 
+    def test_load_from_dir_should_fail_with_invalid_path(self):
+        with self.assertRaises(ValueError) as cm:
+            NLUEngine(engine_dir="/tmp/invalid/path/to/engine")
+
+        self.assertTrue("No such file or directory" in str(cm.exception))
+
     def test_should_load_from_zip_and_parse(self):
         # Given
         engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
@@ -29,12 +35,19 @@ def test_should_load_from_zip_and_parse(self):
         # Then
         self.assertEqual("MakeCoffee", res["intent"]["intentName"])
 
+    def test_load_from_zip_should_fail_with_invalid_data(self):
+        with self.assertRaises(ValueError) as cm:
+            NLUEngine(engine_bytes=bytearray())
+
+        self.assertTrue("Invalid Zip archive" in str(cm.exception))
+
     def test_should_parse_with_whitelist(self):
         # Given
         engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
 
         # Then
-        res = engine.parse("Make me two cups of coffee please", intents_whitelist=["MakeTea"])
+        res = engine.parse("Make me two cups of coffee please",
+                           intents_whitelist=["MakeTea"])
 
         # Then
         self.assertEqual("MakeTea", res["intent"]["intentName"])
@@ -44,7 +57,8 @@ def test_should_parse_with_blacklist(self):
         engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
 
         # Then
-        res = engine.parse("Make me two cups of coffee please", intents_blacklist=["MakeCoffee"])
+        res = engine.parse("Make me two cups of coffee please",
+                           intents_blacklist=["MakeCoffee"])
 
         # Then
         self.assertEqual("MakeTea", res["intent"]["intentName"])
@@ -54,7 +68,8 @@ def test_should_get_slots(self):
         engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
 
         # Then
-        slots = engine.get_slots("Make me two cups of coffee please", intent="MakeCoffee")
+        slots = engine.get_slots("Make me two cups of coffee please",
+                                 intent="MakeCoffee")
 
         # Then
         expected_slots = [
@@ -68,14 +83,32 @@ def test_should_get_slots(self):
         ]
         self.assertEqual(expected_slots, slots)
 
+    def test_get_slots_should_fail_with_unknown_intent(self):
+        # Given
+        engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
+
+        # Then
+        with self.assertRaises(ValueError) as cm:
+            engine.get_slots(
+                "Make me two cups of coffee please", intent="my_intent")
+        self.assertTrue("Unknown intent" in str(cm.exception))
+
     def test_should_get_intents(self):
         # Given
         engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
 
         # Then
-        intents_results = engine.get_intents("Make me two cups of coffee please")
+        intents_results = engine.get_intents(
+            "Make me two cups of coffee please")
         intents = [res["intentName"] for res in intents_results]
 
         # Then
         expected_intents = ["MakeCoffee", "MakeTea", None]
         self.assertEqual(expected_intents, intents)
+
+    def test_engine_should_destroy_itself(self):
+        # Given
+        engine = NLUEngine(engine_bytes=SAMPLE_ENGINE_ZIP_BYTES)
+
+        # When / Then
+        del engine

platforms/python/snips_nlu_rust/utils.py

@@ -1,6 +1,6 @@
-from _ctypes import Structure, POINTER
+from _ctypes import Structure, POINTER, byref
 from contextlib import contextmanager
-from ctypes import cdll, c_char_p, c_int32
+from ctypes import cdll, c_char_p, c_int32, string_at
 from pathlib import Path
 
 dylib_dir = Path(__file__).parent / "dylib"
@@ -21,3 +21,13 @@ class CStringArray(Structure):
         ("data", POINTER(c_char_p)),
         ("size", c_int32)
     ]
+
+
+def check_ffi_error(exit_code, error_context_msg):
+    if exit_code != 0:
+        with string_pointer(c_char_p()) as ptr:
+            if lib.snips_nlu_engine_get_last_error(byref(ptr)) == 0:
+                ffi_error_message = string_at(ptr).decode("utf8")
+            else:
+                ffi_error_message = "see stderr"
+        raise ValueError("%s: %s" % (error_context_msg, ffi_error_message))

platforms/swift/SnipsNlu/Dependencies/build.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-VERSION="0.64.2"
+VERSION="0.64.3"
 SYSTEM=$(echo $1 | tr '[:upper:]' '[:lower:]')
 LIBRARY_NAME=libsnips_nlu_ffi
 LIBRARY_NAME_A=${LIBRARY_NAME}.a