From 939e87e39ebe62ebe1a5edcc3a75cba787a665c2 Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 16:08:57 +0800 Subject: [PATCH 1/6] fix frame precision issue --- maro/backends/backend.pxd | 2 +- maro/backends/np_backend.pyx | 35 +++++++++- maro/backends/raw/attribute.cpp | 3 +- maro/backends/raw/common.h | 2 +- maro/backends/raw/test/README.md | 5 ++ maro/backends/raw/test/main.cpp | 7 ++ maro/backends/raw/test/test_attribute.cpp | 34 +++++++++ maro/backends/raw/test/test_frame.cpp | 84 +++++++++++++++++++++++ maro/backends/raw/xmake.lua | 7 ++ maro/backends/raw_backend.pyx | 70 ++++++++++++++++++- 10 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 maro/backends/raw/test/README.md create mode 100644 maro/backends/raw/test/main.cpp create mode 100644 maro/backends/raw/test/test_attribute.cpp create mode 100644 maro/backends/raw/test/test_frame.cpp create mode 100644 maro/backends/raw/xmake.lua diff --git a/maro/backends/backend.pxd b/maro/backends/backend.pxd index 04fc3c4c9..2af22eff7 100644 --- a/maro/backends/backend.pxd +++ b/maro/backends/backend.pxd @@ -27,7 +27,7 @@ ctypedef float ATTR_FLOAT ctypedef double ATTR_DOUBLE # Type for snapshot querying. -ctypedef float QUERY_FLOAT +ctypedef double QUERY_FLOAT # TYPE of node and attribute ctypedef unsigned short NODE_TYPE diff --git a/maro/backends/np_backend.pyx b/maro/backends/np_backend.pyx index 52e7452f6..c23caeddf 100644 --- a/maro/backends/np_backend.pyx +++ b/maro/backends/np_backend.pyx @@ -30,6 +30,17 @@ attribute_type_mapping = { AttributeType.Double: "d" } +attribute_type_range = { + "b": ("AttributeType.Byte", -128, 127), + "B": ("AttributeType.UByte", 0, 255), + "h": ("AttributeType.Short", -32768, 32767), + "H": ("AttributeType.UShort", 0, 65535), + "i": ("AttributeType.Int", -2147483648, 2147483647), + "I": ("AttributeType.UInt", 0, 4294967295), + "q": ("AttributeType.Long", -9223372036854775808, 9223372036854775807), + "Q": ("AttributeType.ULong", 0, 18446744073709551615), +} + IF NODES_MEMORY_LAYOUT == "ONE_BLOCK": # with this flag, we will allocate a big enough memory for all node types, then use this block construct numpy array @@ -167,6 +178,13 @@ cdef class NumpyBackend(BackendAbc): cdef AttrInfo attr = self._attrs_list[attr_type] + cdef bytes dtype = attr.dtype.encode() + if dtype in attribute_type_range: + assert value >= attribute_type_range[dtype][1] and value <= attribute_type_range[dtype][2], ( + f"Value {value} out of range ({attribute_type_range[dtype][0]}: " + f"[{attribute_type_range[dtype][1]}, {attribute_type_range[dtype][2]}])" + ) + if attr.node_type >= len(self._nodes_list): raise Exception("Invalid node type.") @@ -208,9 +226,22 @@ cdef class NumpyBackend(BackendAbc): cdef AttrInfo attr = self._attrs_list[attr_type] cdef np.ndarray attr_array = self._node_data_dict[attr.node_type][attr.name] + cdef bytes dtype = attr.dtype.encode() + if attr.slot_number == 1: + if dtype in attribute_type_range: + assert value[0] >= attribute_type_range[dtype][1] and value[0] <= attribute_type_range[dtype][2], ( + f"Value {value[0]} out of range ({attribute_type_range[dtype][0]}: " + f"[{attribute_type_range[dtype][1]}, {attribute_type_range[dtype][2]}])" + ) attr_array[0][node_index, slot_index[0]] = value[0] else: + if dtype in attribute_type_range: + for val in value: + assert val >= attribute_type_range[dtype][1] and val <= attribute_type_range[dtype][2], ( + f"Value {val} out of range ({attribute_type_range[dtype][0]}: " + f"[{attribute_type_range[dtype][1]}, {attribute_type_range[dtype][2]}])" + ) attr_array[0][node_index, slot_index] = value cdef list get_attr_values(self, NODE_INDEX node_index, ATTR_TYPE attr_type, SLOT_INDEX[:] slot_indices) except +: @@ -500,10 +531,10 @@ cdef class NPSnapshotList(SnapshotListAbc): # since we have a clear tick to index mapping, do not need additional checking here if tick in self._tick2index_dict: - retq.append(data_arr[attr.name][self._tick2index_dict[tick], node_index].astype("f").flatten()) + retq.append(data_arr[attr.name][self._tick2index_dict[tick], node_index].astype(np.double).flatten()) else: # padding for tick which not exist - retq.append(np.zeros(attr.slot_number, dtype='f')) + retq.append(np.zeros(attr.slot_number, dtype=np.double)) return np.concatenate(retq) diff --git a/maro/backends/raw/attribute.cpp b/maro/backends/raw/attribute.cpp index f8dc1a3a1..5dd8f8afe 100644 --- a/maro/backends/raw/attribute.cpp +++ b/maro/backends/raw/attribute.cpp @@ -61,7 +61,8 @@ namespace maro bool Attribute::is_nan() const noexcept { - return _type == AttrDataType::AFLOAT && isnan(get_value()); + return (_type == AttrDataType::AFLOAT && isnan(get_value())) + || (_type == AttrDataType::ADOUBLE && isnan(get_value())); } template diff --git a/maro/backends/raw/common.h b/maro/backends/raw/common.h index 93d90f17a..cdd52e081 100644 --- a/maro/backends/raw/common.h +++ b/maro/backends/raw/common.h @@ -29,7 +29,7 @@ namespace maro using NODE_INDEX = uint32_t; using SLOT_INDEX = uint32_t; - using QUERY_FLOAT = float; + using QUERY_FLOAT = double; using ATTR_CHAR = char; using ATTR_UCHAR = unsigned char; diff --git a/maro/backends/raw/test/README.md b/maro/backends/raw/test/README.md new file mode 100644 index 000000000..341c5a5d3 --- /dev/null +++ b/maro/backends/raw/test/README.md @@ -0,0 +1,5 @@ +# How to run + +1. Install xmake according to +2. Go to directory: maro/backends/raw +3. Run commands: `xmake; xmake run` diff --git a/maro/backends/raw/test/main.cpp b/maro/backends/raw/test/main.cpp new file mode 100644 index 000000000..f18d5de02 --- /dev/null +++ b/maro/backends/raw/test/main.cpp @@ -0,0 +1,7 @@ +#include + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/maro/backends/raw/test/test_attribute.cpp b/maro/backends/raw/test/test_attribute.cpp new file mode 100644 index 000000000..a403fa498 --- /dev/null +++ b/maro/backends/raw/test/test_attribute.cpp @@ -0,0 +1,34 @@ +#include + +#include "../attribute.h" + +using namespace maro::backends::raw; + + +// test attribute creation +TEST(Attribute, Creation) { + Attribute attr; + + EXPECT_EQ(attr.get_type(), AttrDataType::ACHAR); + EXPECT_FALSE(attr.is_nan()); + EXPECT_EQ(attr.slot_number, 0); + +} + +// test create attribute with other type value. +TEST(Attribute, CreateWithTypedValue) { + Attribute attr{ ATTR_UINT(12)}; + + EXPECT_EQ(attr.get_type(), AttrDataType::AUINT); + EXPECT_EQ(attr.get_value(), 12); + EXPECT_EQ(attr.slot_number, 0); + EXPECT_FALSE(attr.is_nan()); +} + +// test is nan case +TEST(Attribute, CreateWithNan) { + Attribute attr{ nan("nan")}; + + EXPECT_TRUE(attr.is_nan()); +} + diff --git a/maro/backends/raw/test/test_frame.cpp b/maro/backends/raw/test/test_frame.cpp new file mode 100644 index 000000000..f981a5c44 --- /dev/null +++ b/maro/backends/raw/test/test_frame.cpp @@ -0,0 +1,84 @@ +#include +#include + +#include + +#include "../common.h" +#include "../frame.h" +#include "../snapshotlist.h" + +using namespace maro::backends::raw; + + +TEST(test, correct) { + EXPECT_EQ(1, 1); +} + +// show how to use frame and snapshot at c++ end +TEST(test, show_case) { + // a new frame + Frame frame; + + // add a new node with a name + auto node_type = frame.add_node("test_node", 1); + + // add an attribute to this node, this is a list attribute, it has different value to change the value + // NOTE: list means is it dynamic array, that the size can be changed even after setting up + auto attr_type_1 = frame.add_attr(node_type, "a1", AttrDataType::AUINT, 10, false, true); + + // this is a normal attribute + // NOTE: list == false means it is a fixed array that cannot change the size after setting up + auto attr_type_2 = frame.add_attr(node_type, "a2", AttrDataType::AUINT, 2); + + // setup means initialize the frame with node definitions (allocate memory) + // NOTE: call this method before accessing the attributes + frame.setup(); + + // list and normal attribute have different method to set value + frame.set_value(0, attr_type_2, 0, 33554441); + frame.insert_to_list(0, attr_type_1, 0, 33554442); + + // but they have same get method + auto v1 = frame.get_value(0, attr_type_1, 0); + auto v2 = frame.get_value(0, attr_type_2, 0); + + // test with true type + EXPECT_EQ(v2, 33554441); + EXPECT_EQ(v1, 33554442); + + // test with query result type + EXPECT_EQ(QUERY_FLOAT(v2), 3.3554441e+07); + EXPECT_EQ(QUERY_FLOAT(v1), 3.3554442e+07); + + // snapshot instance + SnapshotList ss; + + // NOTE: we need following 2 method to initialize the snapshot instance, or accessing will cause exception + // which frame we will use to copy the values + ss.setup(&frame); + // max snapshot it will keep, oldeat one will be delete when reading the limitation + ss.set_max_size(10); + + // take a snapshot for a tick + ss.take_snapshot(0); + + // query parameters + std::array ticks{ 0 }; + std::array indices{ 0 }; + std::array< ATTR_TYPE, 1> attributes{attr_type_1}; + + // we need use the parameter to get how many items we need to hold the results + auto shape = ss.prepare(node_type, &(ticks[0]), ticks.size(), &(indices[0]), indices.size(), &(attributes[0]), attributes.size()); + + auto total = shape.tick_number * shape.max_node_number * shape.max_slot_number * shape.attr_number; + + // then query (the snapshot instance will remember the latest query parameters, so just pass the result array + QUERY_FLOAT* results = new QUERY_FLOAT[total]; + + ss.query(results); + + // 1st slot value of first node + EXPECT_EQ(results[0], 3.3554442e+07); + + delete[] results; +} diff --git a/maro/backends/raw/xmake.lua b/maro/backends/raw/xmake.lua new file mode 100644 index 000000000..f8643f509 --- /dev/null +++ b/maro/backends/raw/xmake.lua @@ -0,0 +1,7 @@ +add_requires("gtest") + +target("test") + set_kind("binary") + add_files("test/*.cpp") + add_files("./*.cpp") + add_packages("gtest") diff --git a/maro/backends/raw_backend.pyx b/maro/backends/raw_backend.pyx index 584fa0569..0763b7a36 100644 --- a/maro/backends/raw_backend.pyx +++ b/maro/backends/raw_backend.pyx @@ -5,6 +5,8 @@ #distutils: language = c++ #distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION +import warnings + import numpy as np cimport numpy as np cimport cython @@ -288,7 +290,7 @@ cdef class RawSnapshotList(SnapshotListAbc): return None # Result holder - cdef QUERY_FLOAT[:, :, :, :] result = view.array(shape=(shape.tick_number, shape.max_node_number, shape.attr_number, shape.max_slot_number), itemsize=sizeof(QUERY_FLOAT), format="f") + cdef QUERY_FLOAT[:, :, :, :] result = view.array(shape=(shape.tick_number, shape.max_node_number, shape.attr_number, shape.max_slot_number), itemsize=sizeof(QUERY_FLOAT), format="d") # Default result value result[:, :, :, :] = 0 @@ -331,139 +333,205 @@ cdef class RawSnapshotList(SnapshotListAbc): cdef class AttributeCharAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -128 and value <= 127, f"Value {value} out of range (AttributeType.Byte: [-127, 128])" self._backend._frame.set_value[ATTR_CHAR](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_CHAR](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= -128 and value <= 127, f"Value {value} out of range (AttributeType.Byte: [-127, 128])" self._backend._frame.append_to_list[ATTR_CHAR](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -128 and value <= 127, f"Value {value} out of range (AttributeType.Byte: [-127, 128])" self._backend._frame.insert_to_list[ATTR_CHAR](node_index, self._attr_type, slot_index, value) cdef class AttributeUCharAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 255, f"Value {value} out of range (AttributeType.UByte: [0, 255])" self._backend._frame.set_value[ATTR_UCHAR](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_UCHAR](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= 0 and value <= 255, f"Value {value} out of range (AttributeType.UByte: [0, 255])" self._backend._frame.append_to_list[ATTR_UCHAR](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 255, f"Value {value} out of range (AttributeType.UByte: [0, 255])" self._backend._frame.insert_to_list[ATTR_UCHAR](node_index, self._attr_type, slot_index, value) cdef class AttributeShortAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -32768 and value <= 32767, ( + f"Value {value} out of range (AttributeType.Short: [-32,768, 32,767])" + ) self._backend._frame.set_value[ATTR_SHORT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_SHORT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= -32768 and value <= 32767, ( + f"Value {value} out of range (AttributeType.Short: [-32,768, 32,767])" + ) self._backend._frame.append_to_list[ATTR_SHORT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -32768 and value <= 32767, ( + f"Value {value} out of range (AttributeType.Short: [-32,768, 32,767])" + ) self._backend._frame.insert_to_list[ATTR_SHORT](node_index, self._attr_type, slot_index, value) cdef class AttributeUShortAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 65535, f"Value {value} out of range (AttributeType.UShort: [0, 65,535])" self._backend._frame.set_value[ATTR_USHORT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_USHORT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= 0 and value <= 65535, f"Value {value} out of range (AttributeType.UShort: [0, 65,535])" self._backend._frame.append_to_list[ATTR_USHORT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 65535, f"Value {value} out of range (AttributeType.UShort: [0, 65,535])" self._backend._frame.insert_to_list[ATTR_USHORT](node_index, self._attr_type, slot_index, value) cdef class AttributeIntAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -2147483648 and value <= 2147483647, ( + f"Value {value} out of range (AttributeType.Int: [-2,147,483,648, 2,147,483,647])" + ) self._backend._frame.set_value[ATTR_INT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_INT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= -2147483648 and value <= 2147483647, ( + f"Value {value} out of range (AttributeType.Int: [-2,147,483,648, 2,147,483,647])" + ) self._backend._frame.append_to_list[ATTR_INT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -2147483648 and value <= 2147483647, ( + f"Value {value} out of range (AttributeType.Int: [-2,147,483,648, 2,147,483,647])" + ) self._backend._frame.insert_to_list[ATTR_INT](node_index, self._attr_type, slot_index, value) cdef class AttributeUIntAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 4294967295, ( + f"Value {value} out of range (AttributeType.UInt: [0, 4,294,967,295])" + ) self._backend._frame.set_value[ATTR_UINT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_UINT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= 0 and value <= 4294967295, ( + f"Value {value} out of range (AttributeType.UInt: [0, 4,294,967,295])" + ) self._backend._frame.append_to_list[ATTR_UINT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 4294967295, ( + f"Value {value} out of range (AttributeType.UInt: [0, 4,294,967,295])" + ) self._backend._frame.insert_to_list[ATTR_UINT](node_index, self._attr_type, slot_index, value) cdef class AttributeLongAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -9223372036854775808 and value <= 9223372036854775807, ( + f"Value {value} out of range (AttributeType.Long: [-9,223,372,036,854,775,808, 9,223,372,036,854,775,807])" + ) self._backend._frame.set_value[ATTR_LONG](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_LONG](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= -9223372036854775808 and value <= 9223372036854775807, ( + f"Value {value} out of range (AttributeType.Long: [-9,223,372,036,854,775,808, 9,223,372,036,854,775,807])" + ) self._backend._frame.append_to_list[ATTR_LONG](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= -9223372036854775808 and value <= 9223372036854775807, ( + f"Value {value} out of range (AttributeType.Long: [-9,223,372,036,854,775,808, 9,223,372,036,854,775,807])" + ) self._backend._frame.insert_to_list[ATTR_LONG](node_index, self._attr_type, slot_index, value) cdef class AttributeULongAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 18446744073709551615, ( + f"Value {value} out of range (AttributeType.ULong: [0, 18,446,744,073,709,551,615])" + ) self._backend._frame.set_value[ATTR_ULONG](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_ULONG](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + assert value >= 0 and value <= 18446744073709551615, ( + f"Value {value} out of range (AttributeType.ULong: [0, 18,446,744,073,709,551,615])" + ) self._backend._frame.append_to_list[ATTR_ULONG](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + assert value >= 0 and value <= 18446744073709551615, ( + f"Value {value} out of range (AttributeType.ULong: [0, 18,446,744,073,709,551,615])" + ) self._backend._frame.insert_to_list[ATTR_ULONG](node_index, self._attr_type, slot_index, value) cdef class AttributeFloatAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + if abs(float(f"{value:e}") - value) > 0.001: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") self._backend._frame.set_value[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_FLOAT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + if abs(float(f"{value:e}") - value) > 0.001: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") self._backend._frame.append_to_list[ATTR_FLOAT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + if abs(float(f"{value:e}") - value) > 0.001: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") self._backend._frame.insert_to_list[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) cdef class AttributeDoubleAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + if float(f"{value:.15e}") != value: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") self._backend._frame.set_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: + if float(f"{value:.15e}") != value: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") self._backend._frame.append_to_list[ATTR_DOUBLE](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: + if float(f"{value:.15e}") != value: + warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") self._backend._frame.insert_to_list[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) From 8237052e1cf7563d9996aa6de7f568880923d77a Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 16:09:33 +0800 Subject: [PATCH 2/6] add .xmake to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4ec4f4bd6..2d5f5ea48 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ htmlcov/ .coverage .coveragerc .tmp/ +.xmake/ From 5f095fcb20504cb102bd05c4dd51fc06b8ce8a8f Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 16:52:16 +0800 Subject: [PATCH 3/6] update frame precision lost warning message --- maro/backends/raw_backend.pyx | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/maro/backends/raw_backend.pyx b/maro/backends/raw_backend.pyx index 0763b7a36..43683ab30 100644 --- a/maro/backends/raw_backend.pyx +++ b/maro/backends/raw_backend.pyx @@ -499,39 +499,45 @@ cdef class AttributeULongAccessor(AttributeAccessor): cdef class AttributeFloatAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: - if abs(float(f"{value:e}") - value) > 0.001: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") + n_val = float(f"{value:e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.set_value[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_FLOAT](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: - if abs(float(f"{value:e}") - value) > 0.001: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") + n_val = float(f"{value:e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.append_to_list[ATTR_FLOAT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: - if abs(float(f"{value:e}") - value) > 0.001: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Float precision: .9") + n_val = float(f"{value:e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.insert_to_list[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) cdef class AttributeDoubleAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: - if float(f"{value:.15e}") != value: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") + n_val = float(f"{value:.15e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.set_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: return self._backend._frame.get_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index) cdef void append_value(self, NODE_INDEX node_index, object value) except +: - if float(f"{value:.15e}") != value: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") + n_val = float(f"{value:.15e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.append_to_list[ATTR_DOUBLE](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: - if float(f"{value:.15e}") != value: - warnings.warn(f"[Precision lost] Value {value} out of AttributeType.Double precision: .15") + n_val = float(f"{value:.15e}") + if n_val != value: + warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.insert_to_list[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) From 11e8916a2e836f22bc8147fcbe279881fd37d50e Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 17:04:17 +0800 Subject: [PATCH 4/6] add assert to frame precision checking --- maro/backends/raw_backend.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/maro/backends/raw_backend.pyx b/maro/backends/raw_backend.pyx index 43683ab30..b7dba4b71 100644 --- a/maro/backends/raw_backend.pyx +++ b/maro/backends/raw_backend.pyx @@ -500,6 +500,7 @@ cdef class AttributeULongAccessor(AttributeAccessor): cdef class AttributeFloatAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.set_value[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) @@ -509,12 +510,14 @@ cdef class AttributeFloatAccessor(AttributeAccessor): cdef void append_value(self, NODE_INDEX node_index, object value) except +: n_val = float(f"{value:e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.append_to_list[ATTR_FLOAT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.insert_to_list[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) @@ -523,6 +526,7 @@ cdef class AttributeFloatAccessor(AttributeAccessor): cdef class AttributeDoubleAccessor(AttributeAccessor): cdef void set_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:.15e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.set_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) @@ -532,12 +536,14 @@ cdef class AttributeDoubleAccessor(AttributeAccessor): cdef void append_value(self, NODE_INDEX node_index, object value) except +: n_val = float(f"{value:.15e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.append_to_list[ATTR_DOUBLE](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:.15e}") + assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") self._backend._frame.insert_to_list[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) From d495860e6bfd1d32de7facf76afa32a5314534c8 Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 17:07:40 +0800 Subject: [PATCH 5/6] typo fix --- maro/backends/raw_backend.pyx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/maro/backends/raw_backend.pyx b/maro/backends/raw_backend.pyx index b7dba4b71..0d7195f64 100644 --- a/maro/backends/raw_backend.pyx +++ b/maro/backends/raw_backend.pyx @@ -502,7 +502,7 @@ cdef class AttributeFloatAccessor(AttributeAccessor): n_val = float(f"{value:e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.set_value[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: @@ -512,14 +512,14 @@ cdef class AttributeFloatAccessor(AttributeAccessor): n_val = float(f"{value:e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.append_to_list[ATTR_FLOAT](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Float)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.insert_to_list[ATTR_FLOAT](node_index, self._attr_type, slot_index, value) @@ -528,7 +528,7 @@ cdef class AttributeDoubleAccessor(AttributeAccessor): n_val = float(f"{value:.15e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.set_value[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) cdef object get_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index) except +: @@ -538,12 +538,12 @@ cdef class AttributeDoubleAccessor(AttributeAccessor): n_val = float(f"{value:.15e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.append_to_list[ATTR_DOUBLE](node_index, self._attr_type, value) cdef void insert_value(self, NODE_INDEX node_index, SLOT_INDEX slot_index, object value) except +: n_val = float(f"{value:.15e}") assert abs(n_val - value) < 1, f"Value {value} out of range (AttributeType.Double)" if n_val != value: - warnings.warn(f"[Precision lost] Value {value} would be convert to {n_val}") + warnings.warn(f"[Precision lost] Value {value} would be converted to {n_val}") self._backend._frame.insert_to_list[ATTR_DOUBLE](node_index, self._attr_type, slot_index, value) From 0aeecb3dcb514d1d5974fb474068bf849917c5e1 Mon Sep 17 00:00:00 2001 From: Jinyu Date: Thu, 9 Jun 2022 17:49:23 +0800 Subject: [PATCH 6/6] add TODO for future Long data type issue fix --- maro/backends/raw/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/backends/raw/common.h b/maro/backends/raw/common.h index cdd52e081..89c975c4e 100644 --- a/maro/backends/raw/common.h +++ b/maro/backends/raw/common.h @@ -29,7 +29,7 @@ namespace maro using NODE_INDEX = uint32_t; using SLOT_INDEX = uint32_t; - using QUERY_FLOAT = double; + using QUERY_FLOAT = double; // TODO: Precision issue for Long data type. using ATTR_CHAR = char; using ATTR_UCHAR = unsigned char;