Merge branch 'master' of https://github.com/xuepanchen/arrow

c_glib/configure.ac

@@ -143,7 +143,7 @@ AC_CONFIG_FILES([
   arrow-gpu-glib/arrow-gpu-glib.pc
   doc/Makefile
   doc/reference/Makefile
-  doc/reference/xml/Makefile
+  doc/reference/entities.xml
   example/Makefile
   example/lua/Makefile
   tool/Makefile

c_glib/doc/reference/Makefile.am

@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-SUBDIRS =                                       \
-	xml
-
 DOC_MODULE = arrow-glib
 
 DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.xml
@@ -72,4 +69,5 @@ CLEANFILES +=					\
 	$(DOC_MODULE).types
 
 EXTRA_DIST +=					\
+	entities.xml.in				\
 	meson.build

c_glib/doc/reference/arrow-glib-docs.xml

@@ -21,10 +21,10 @@
                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
 [
   <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
-  <!ENTITY % gtkdocentities SYSTEM "xml/gtkdocentities.ent">
+  <!ENTITY % gtkdocentities SYSTEM "entities.xml">
   %gtkdocentities;
 ]>
-<book id="index">
+<book id="index" xmlns:xi="http://www.w3.org/2003/XInclude">
   <bookinfo>
     <title>&package_name; Reference Manual</title>
     <releaseinfo>

c_glib/doc/reference/xml/gtkdocentities.ent.in → c_glib/doc/reference/entities.xml.in

@@ -16,9 +16,9 @@
   specific language governing permissions and limitations
   under the License.
 -->
-<!ENTITY package "@package@">
-<!ENTITY package_bugreport "@package_bugreport@">
-<!ENTITY package_name "@package_name@">
-<!ENTITY package_string "@package_string@">
-<!ENTITY package_url "@package_url@">
-<!ENTITY package_version "@package_version@">
+<!ENTITY package "@PACKAGE@">
+<!ENTITY package_bugreport "@PACKAGE_BUGREPORT@">
+<!ENTITY package_name "@PACKAGE_NAME@">
+<!ENTITY package_string "@PACKAGE_STRING@">
+<!ENTITY package_url "@PACKAGE_URL@">
+<!ENTITY package_version "@PACKAGE_VERSION@">

c_glib/doc/reference/meson.build

@@ -17,7 +17,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-subdir('xml')
+entities_conf = configuration_data()
+entities_conf.set('PACKAGE', meson.project_name())
+entities_conf.set('PACKAGE_BUGREPORT',
+                  'https://issues.apache.org/jira/browse/ARROW')
+entities_conf.set('PACKAGE_NAME', meson.project_name())
+entities_conf.set('PACKAGE_STRING',
+                  ' '.join([meson.project_name(), version]))
+entities_conf.set('PACKAGE_URL', 'https://arrow.apache.org/')
+entities_conf.set('PACKAGE_VERSION', version)
+configure_file(input: 'entities.xml.in',
+               output: 'entities.xml',
+               configuration: entities_conf)
 
 private_headers = [
 ]

ci/travis_lint.sh

@@ -35,10 +35,10 @@ popd
 # Fail fast on style checks
 sudo pip install flake8
 
-PYARROW_DIR=$TRAVIS_BUILD_DIR/python/pyarrow
+PYTHON_DIR=$TRAVIS_BUILD_DIR/python
 
-flake8 --count $PYARROW_DIR
+flake8 --count $PYTHON_DIR/pyarrow
 
 # Check Cython files with some checks turned off
 flake8 --count --config=$PYTHON_DIR/.flake8.cython \
-       $PYARROW_DIR
+       $PYTHON_DIR/pyarrow

cpp/src/arrow/python/numpy_to_arrow.cc

@@ -175,7 +175,7 @@ static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
       continue;
     } else if (!PyBytes_Check(obj)) {
       std::stringstream ss;
-      ss << "Error converting to Python objects to bytes: ";
+      ss << "Error converting from Python objects to bytes: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }
@@ -230,7 +230,7 @@ static Status AppendObjectStrings(PyArrayObject* arr, PyArrayObject* mask, int64
       *have_bytes = true;
     } else {
       std::stringstream ss;
-      ss << "Error converting to Python objects to String/UTF8: ";
+      ss << "Error converting from Python objects to String/UTF8: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }
@@ -278,7 +278,7 @@ static Status AppendObjectFixedWidthBytes(PyArrayObject* arr, PyArrayObject* mas
       tmp_obj.reset(obj);
     } else if (!PyBytes_Check(obj)) {
       std::stringstream ss;
-      ss << "Error converting to Python objects to FixedSizeBinary: ";
+      ss << "Error converting from Python objects to FixedSizeBinary: ";
       RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
       return Status::Invalid(ss.str());
     }

cpp/src/arrow/table-test.cc

@@ -108,6 +108,21 @@ TEST_F(TestChunkedArray, EqualsDifferingLengths) {
   ASSERT_TRUE(one_->Equals(*another_.get()));
 }
 
+TEST_F(TestChunkedArray, SliceEquals) {
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(100));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  Construct();
+
+  std::shared_ptr<ChunkedArray> slice = one_->Slice(125, 50);
+  ASSERT_EQ(slice->length(), 50);
+  ASSERT_TRUE(slice->Equals(one_->Slice(125, 50)));
+
+  std::shared_ptr<ChunkedArray> slice2 = one_->Slice(75)->Slice(25)->Slice(25, 50);
+  ASSERT_EQ(slice2->length(), 50);
+  ASSERT_TRUE(slice2->Equals(slice));
+}
+
 class TestColumn : public TestChunkedArray {
  protected:
   void Construct() override {
@@ -158,6 +173,22 @@ TEST_F(TestColumn, ChunksInhomogeneous) {
   ASSERT_RAISES(Invalid, column_->ValidateData());
 }
 
+TEST_F(TestColumn, SliceEquals) {
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(100));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  arrays_one_.push_back(MakeRandomArray<Int32Array>(50));
+  one_field_ = field("column", int32());
+  Construct();
+
+  std::shared_ptr<Column> slice = one_col_->Slice(125, 50);
+  ASSERT_EQ(slice->length(), 50);
+  ASSERT_TRUE(slice->Equals(one_col_->Slice(125, 50)));
+
+  std::shared_ptr<Column> slice2 = one_col_->Slice(75)->Slice(25)->Slice(25, 50);
+  ASSERT_EQ(slice2->length(), 50);
+  ASSERT_TRUE(slice2->Equals(slice));
+}
+
 TEST_F(TestColumn, Equals) {
   std::vector<bool> null_bitmap(100, true);
   std::vector<int32_t> data(100, 1);

cpp/src/arrow/table.cc

@@ -102,6 +102,30 @@ bool ChunkedArray::Equals(const std::shared_ptr<ChunkedArray>& other) const {
   return Equals(*other.get());
 }
 
+std::shared_ptr<ChunkedArray> ChunkedArray::Slice(int64_t offset, int64_t length) const {
+  DCHECK_LE(offset, length_);
+
+  int curr_chunk = 0;
+  while (offset >= chunk(curr_chunk)->length()) {
+    offset -= chunk(curr_chunk)->length();
+    curr_chunk++;
+  }
+
+  ArrayVector new_chunks;
+  while (length > 0 && curr_chunk < num_chunks()) {
+    new_chunks.push_back(chunk(curr_chunk)->Slice(offset, length));
+    length -= chunk(curr_chunk)->length() - offset;
+    offset = 0;
+    curr_chunk++;
+  }
+
+  return std::make_shared<ChunkedArray>(new_chunks);
+}
+
+std::shared_ptr<ChunkedArray> ChunkedArray::Slice(int64_t offset) const {
+  return Slice(offset, length_);
+}
+
 Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks)
     : field_(field) {
   data_ = std::make_shared<ChunkedArray>(chunks);

cpp/src/arrow/table.h

@@ -44,6 +44,7 @@ class ARROW_EXPORT ChunkedArray {
   /// \return the total length of the chunked array; computed on construction
   int64_t length() const { return length_; }
 
+  /// \return the total number of nulls among all chunks
   int64_t null_count() const { return null_count_; }
 
   int num_chunks() const { return static_cast<int>(chunks_.size()); }
@@ -53,6 +54,20 @@ class ARROW_EXPORT ChunkedArray {
 
   const ArrayVector& chunks() const { return chunks_; }
 
+  /// \brief Construct a zero-copy slice of the chunked array with the
+  /// indicated offset and length
+  ///
+  /// \param[in] offset the position of the first element in the constructed
+  /// slice
+  /// \param[in] length the length of the slice. If there are not enough
+  /// elements in the chunked array, the length will be adjusted accordingly
+  ///
+  /// \return a new object wrapped in std::shared_ptr<ChunkedArray>
+  std::shared_ptr<ChunkedArray> Slice(int64_t offset, int64_t length) const;
+
+  /// \brief Slice from offset until end of the chunked array
+  std::shared_ptr<ChunkedArray> Slice(int64_t offset) const;
+
   std::shared_ptr<DataType> type() const;
 
   bool Equals(const ChunkedArray& other) const;
@@ -67,8 +82,9 @@ class ARROW_EXPORT ChunkedArray {
   ARROW_DISALLOW_COPY_AND_ASSIGN(ChunkedArray);
 };
 
+/// \class Column
 /// \brief An immutable column data structure consisting of a field (type
-/// metadata) and a logical chunked data array
+/// metadata) and a chunked data array
 class ARROW_EXPORT Column {
  public:
   Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
@@ -97,6 +113,24 @@ class ARROW_EXPORT Column {
   /// \return the column's data as a chunked logical array
   std::shared_ptr<ChunkedArray> data() const { return data_; }
 
+  /// \brief Construct a zero-copy slice of the column with the indicated
+  /// offset and length
+  ///
+  /// \param[in] offset the position of the first element in the constructed
+  /// slice
+  /// \param[in] length the length of the slice. If there are not enough
+  /// elements in the column, the length will be adjusted accordingly
+  ///
+  /// \return a new object wrapped in std::shared_ptr<Column>
+  std::shared_ptr<Column> Slice(int64_t offset, int64_t length) const {
+    return std::make_shared<Column>(field_, data_->Slice(offset, length));
+  }
+
+  /// \brief Slice from offset until end of the column
+  std::shared_ptr<Column> Slice(int64_t offset) const {
+    return std::make_shared<Column>(field_, data_->Slice(offset));
+  }
+
   bool Equals(const Column& other) const;
   bool Equals(const std::shared_ptr<Column>& other) const;