diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..9b9baf9f47 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# .git-blame-ignore-revs +# Format code using the new clang-format version +1aac45490f8192cded9e3656a215935c92f331fa \ No newline at end of file diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 30b1d79bc2..89e4cfcea6 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -11,9 +11,3 @@ jobs: - name: Run pre-commit hooks on all files run: python -m pip -v install pre-commit && pre-commit run -a -v - - - name: Check formatting (linux only)' - run: | - source $GITHUB_WORKSPACE/ci/scripts/check_formatting_linux.sh - shell: bash - if: ${{ runner.os == 'Linux' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3fd8e17ca..141feaa8ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,3 +7,10 @@ repos: rev: v0.4.4 hooks: - id: ruff + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.8 + hooks: + - id: clang-format + args: ["-style=file"] + types_or: [c, c++] + files: tiledb/ diff --git a/ci/scripts/check_formatting_linux.sh b/ci/scripts/check_formatting_linux.sh deleted file mode 100644 index 07b61f61f1..0000000000 --- a/ci/scripts/check_formatting_linux.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# -# The MIT License (MIT) -# -# Copyright (c) 2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - -# Check test formatting - linux ONLY - -set -e pipefail - -# Install clang-format -ls -la -sudo ./ci/scripts/install-clangformat.sh - -src=$GITHUB_WORKSPACE -cd $src - -$src/ci/scripts/run-clang-format.sh $src clang-format-16 0 diff --git a/ci/scripts/install-clangformat.sh b/ci/scripts/install-clangformat.sh deleted file mode 100755 index 3bb230b8a5..0000000000 --- a/ci/scripts/install-clangformat.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# -# The MIT License (MIT) -# -# Copyright (c) 2019-2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# - -die() { - echo "$@" 1>&2 ; popd 2>/dev/null; exit 1 -} - -install_apt_pkg() { - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main' && - apt-get update -qq && apt-get install -qq -y clang-format-16 -} - -install_brew_pkg() { - brew upgrade && brew install clang-format -} - -install_clang_format() { - if [[ $OSTYPE == linux* ]]; then - if [ -n "$(command -v apt-get)" ]; then - install_apt_pkg || die "could not install apt clang format package" - else - die "unsupported Linux package management system" - fi - elif [[ $OSTYPE == darwin* ]]; then - if [ -n "$(command -v brew)" ]; then - install_brew_pkg || die "could not install brew clang format package" - else - die "homebrew is not installed!" - fi - else - die "unsupported OS" - fi -} - -run() { - install_clang_format -} - -run diff --git a/ci/scripts/run-clang-format.sh b/ci/scripts/run-clang-format.sh deleted file mode 100755 index 1249ddaa0e..0000000000 --- a/ci/scripts/run-clang-format.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Runs clang format in the given directory -# Arguments: -# $1 - Path to the source tree -# $2 - Path to the clang format binary -# $3 - Apply fixes (will raise an error if false and not there where changes) -# $ARGN - Files to run clang format on -# - -SOURCE_DIR=$1 -shift -CLANG_FORMAT=$1 -shift -APPLY_FIXES=$1 -shift - -echo "Running clang-format version:" `$CLANG_FORMAT --version` - -# clang format will only find its configuration if we are in -# the source tree or in a path relative to the source tree -pushd $SOURCE_DIR - -src=$SOURCE_DIR -SOURCE_PATHS=($src/tiledb) -FIND_FILES=(-name "*.cc" -or -name "*.c" -or -name "*.h") - -if [ "$APPLY_FIXES" == "1" ]; then - find "${SOURCE_PATHS[@]}" \( "${FIND_FILES[@]}" \) -print0 | xargs -0 -P8 $CLANG_FORMAT -i - -else - NUM_CORRECTIONS=`find "${SOURCE_PATHS[@]}" \( "${FIND_FILES[@]}" \) -print0 | xargs -0 -P8 $CLANG_FORMAT -output-replacements-xml | grep offset | wc -l` - - if [ "$NUM_CORRECTIONS" -gt "0" ]; then - echo "clang-format suggested changes" - - # If running on CI, print out the change-set - if [ "$CI" = true ]; then - echo "-------- see list of files to update below --------" - find "${SOURCE_PATHS[@]}" "${FIND_FILES[@]}" -print0 | xargs -P8 -0 $CLANG_FORMAT -i - git diff --name-only - - echo "-------- see diff below --------" - git diff - fi - - # Fail the job - exit 1 - fi -fi -popd diff --git a/tiledb/cc/array.cc b/tiledb/cc/array.cc index 89d51cb0e3..758f2df535 100644 --- a/tiledb/cc/array.cc +++ b/tiledb/cc/array.cc @@ -1,5 +1,5 @@ -#include // for enums -#include // C++ +#include // for enums +#include // C++ #include "common.h" @@ -13,20 +13,22 @@ namespace libtiledbcpp { using namespace tiledb; namespace py = pybind11; -void init_array(py::module &m) { +void init_array(py::module& m) { py::class_(m, "Array") //.def(py::init()) .def( - py::init(), + py::init(), py::keep_alive<1, 2>() /* Array keeps Context alive */) // Temporary initializer while Array is converted from Cython to PyBind. - .def(py::init([](const Context &ctx, py::object array) { - tiledb_array_t *c_array = (py::capsule)array.attr("__capsule__")(); - return std::make_unique(ctx, c_array, false); - }), - py::keep_alive<1, 2>(), py::keep_alive<1, 3>()) + .def( + py::init([](const Context& ctx, py::object array) { + tiledb_array_t* c_array = (py::capsule)array.attr("__capsule__")(); + return std::make_unique(ctx, c_array, false); + }), + py::keep_alive<1, 2>(), + py::keep_alive<1, 3>()) // TODO capsule Array(const Context& ctx, tiledb_array_t* carray, // tiledb_config_t* config) @@ -38,155 +40,185 @@ void init_array(py::module &m) { .def("reopen", &Array::reopen) .def("set_open_timestamp_start", &Array::set_open_timestamp_start) .def("set_open_timestamp_end", &Array::set_open_timestamp_end) - .def_property_readonly("open_timestamp_start", - &Array::open_timestamp_start) + .def_property_readonly( + "open_timestamp_start", &Array::open_timestamp_start) .def_property_readonly("open_timestamp_end", &Array::open_timestamp_end) .def("set_config", &Array::set_config) .def("config", &Array::config) .def("close", &Array::close) - .def("consolidate", - [](Array &self, const Context &ctx, Config *config) { - if (self.query_type() == TILEDB_READ) { - throw TileDBError("cannot consolidate array opened in readonly " - "mode (mode='r')"); - } - Array::consolidate(ctx, self.uri(), config); - }) - .def("consolidate", - [](Array &self, const Context &ctx, - const std::vector &fragment_uris, Config *config) { - if (self.query_type() == TILEDB_READ) { - throw TileDBError("cannot consolidate array opened in readonly " - "mode (mode='r')"); - } - std::vector c_strings; - c_strings.reserve(fragment_uris.size()); - for (const auto &str : fragment_uris) { - c_strings.push_back(str.c_str()); - } - - Array::consolidate(ctx, self.uri(), c_strings.data(), - fragment_uris.size(), config); - }) - .def("consolidate", - [](Array &self, const Context &ctx, - const std::tuple ×tamp, Config *config) { - if (self.query_type() == TILEDB_READ) { - throw TileDBError("cannot consolidate array opened in readonly " - "mode (mode='r')"); - } - int start, end; - std::tie(start, end) = timestamp; - - config->set("sm.consolidation.timestamp_start", - std::to_string(start)); - config->set("sm.consolidation.timestamp_end", std::to_string(end)); - - Array::consolidate(ctx, self.uri(), config); - }) + .def( + "consolidate", + [](Array& self, const Context& ctx, Config* config) { + if (self.query_type() == TILEDB_READ) { + throw TileDBError( + "cannot consolidate array opened in readonly " + "mode (mode='r')"); + } + Array::consolidate(ctx, self.uri(), config); + }) + .def( + "consolidate", + [](Array& self, + const Context& ctx, + const std::vector& fragment_uris, + Config* config) { + if (self.query_type() == TILEDB_READ) { + throw TileDBError( + "cannot consolidate array opened in readonly " + "mode (mode='r')"); + } + std::vector c_strings; + c_strings.reserve(fragment_uris.size()); + for (const auto& str : fragment_uris) { + c_strings.push_back(str.c_str()); + } + + Array::consolidate( + ctx, + self.uri(), + c_strings.data(), + fragment_uris.size(), + config); + }) + .def( + "consolidate", + [](Array& self, + const Context& ctx, + const std::tuple& timestamp, + Config* config) { + if (self.query_type() == TILEDB_READ) { + throw TileDBError( + "cannot consolidate array opened in readonly " + "mode (mode='r')"); + } + int start, end; + std::tie(start, end) = timestamp; + + config->set( + "sm.consolidation.timestamp_start", std::to_string(start)); + config->set("sm.consolidation.timestamp_end", std::to_string(end)); + + Array::consolidate(ctx, self.uri(), config); + }) .def("vacuum", &Array::vacuum) - .def("create", - py::overload_cast( - &Array::create)) - .def("load_schema", - py::overload_cast( - &Array::load_schema)) + .def( + "create", + py::overload_cast( + &Array::create)) + .def( + "load_schema", + py::overload_cast( + &Array::load_schema)) .def("encryption_type", &Array::encryption_type) // TODO non_empty_domain // TODO non_empty_domain_var .def("query_type", &Array::query_type) - .def("consolidate_fragments", - [](Array &self, const Context &ctx, - const std::vector &fragment_uris, Config *config) { - std::vector c_strings; - c_strings.reserve(fragment_uris.size()); - for (const auto &str : fragment_uris) { - c_strings.push_back(str.c_str()); - } - ctx.handle_error(tiledb_array_consolidate_fragments( - ctx.ptr().get(), self.uri().c_str(), c_strings.data(), - fragment_uris.size(), config->ptr().get())); - }) - .def("consolidate_metadata", - py::overload_cast(&Array::consolidate_metadata)) - .def("put_metadata", - [](Array &self, std::string &key, tiledb_datatype_t tdb_type, - const py::buffer &b) { - py::buffer_info info = b.request(); - - // size_t size = std::reduce(info.shape.begin(), - // info.shape.end()); - size_t size = 1; - for (auto s : info.shape) { - size *= s; - } - // size_t nbytes = size * info.itemsize; - - self.put_metadata(key, tdb_type, size, info.ptr); - /* - std::cout << "ndim: " << info.ndim << std::endl; - - - std::cout << "sz: " << size << std::endl; - std::cout << "imsz: " << info.itemsize << std::endl; - - std::cout << "--|" << std::endl; - for (auto& s : info.shape) { - std::cout << s << std::endl; - } - */ - }) - .def("get_metadata", - [](Array &self, std::string &key) -> py::buffer { - tiledb_datatype_t tdb_type; - uint32_t value_num = 0; - const void *data_ptr = nullptr; - - self.get_metadata(key, &tdb_type, &value_num, &data_ptr); - - if (data_ptr == nullptr && value_num != 1) { - throw py::key_error(); - } - - assert(data_ptr != nullptr); - return py::memoryview::from_memory( - data_ptr, value_num * tiledb_datatype_size(tdb_type)); - }) - .def("get_metadata_from_index", - [](Array &self, uint64_t index) -> py::tuple { - tiledb_datatype_t tdb_type; - uint32_t value_num = 0; - const void *data_ptr = nullptr; - std::string key; - - self.get_metadata_from_index(index, &key, &tdb_type, &value_num, - &data_ptr); - - if (data_ptr == nullptr && value_num != 1) { - throw py::key_error(); - } - // TODO handle empty value case - - assert(data_ptr != nullptr); - auto buf = py::memoryview::from_memory( - data_ptr, value_num * tiledb_datatype_size(tdb_type)); - - return py::make_tuple(tdb_type, buf); - }) + .def( + "consolidate_fragments", + [](Array& self, + const Context& ctx, + const std::vector& fragment_uris, + Config* config) { + std::vector c_strings; + c_strings.reserve(fragment_uris.size()); + for (const auto& str : fragment_uris) { + c_strings.push_back(str.c_str()); + } + ctx.handle_error(tiledb_array_consolidate_fragments( + ctx.ptr().get(), + self.uri().c_str(), + c_strings.data(), + fragment_uris.size(), + config->ptr().get())); + }) + .def( + "consolidate_metadata", + py::overload_cast( + &Array::consolidate_metadata)) + .def( + "put_metadata", + [](Array& self, + std::string& key, + tiledb_datatype_t tdb_type, + const py::buffer& b) { + py::buffer_info info = b.request(); + + // size_t size = std::reduce(info.shape.begin(), + // info.shape.end()); + size_t size = 1; + for (auto s : info.shape) { + size *= s; + } + // size_t nbytes = size * info.itemsize; + + self.put_metadata(key, tdb_type, size, info.ptr); + /* + std::cout << "ndim: " << info.ndim << std::endl; + + + std::cout << "sz: " << size << std::endl; + std::cout << "imsz: " << info.itemsize << std::endl; + + std::cout << "--|" << std::endl; + for (auto& s : info.shape) { + std::cout << s << std::endl; + } + */ + }) + .def( + "get_metadata", + [](Array& self, std::string& key) -> py::buffer { + tiledb_datatype_t tdb_type; + uint32_t value_num = 0; + const void* data_ptr = nullptr; + + self.get_metadata(key, &tdb_type, &value_num, &data_ptr); + + if (data_ptr == nullptr && value_num != 1) { + throw py::key_error(); + } + + assert(data_ptr != nullptr); + return py::memoryview::from_memory( + data_ptr, value_num * tiledb_datatype_size(tdb_type)); + }) + .def( + "get_metadata_from_index", + [](Array& self, uint64_t index) -> py::tuple { + tiledb_datatype_t tdb_type; + uint32_t value_num = 0; + const void* data_ptr = nullptr; + std::string key; + + self.get_metadata_from_index( + index, &key, &tdb_type, &value_num, &data_ptr); + + if (data_ptr == nullptr && value_num != 1) { + throw py::key_error(); + } + // TODO handle empty value case + + assert(data_ptr != nullptr); + auto buf = py::memoryview::from_memory( + data_ptr, value_num * tiledb_datatype_size(tdb_type)); + + return py::make_tuple(tdb_type, buf); + }) .def("delete_metadata", &Array::delete_metadata) - .def("has_metadata", - [](Array &self, std::string &key) -> py::tuple { - tiledb_datatype_t has_type; - bool has_it = self.has_metadata(key, &has_type); - return py::make_tuple(has_it, has_type); - }) + .def( + "has_metadata", + [](Array& self, std::string& key) -> py::tuple { + tiledb_datatype_t has_type; + bool has_it = self.has_metadata(key, &has_type); + return py::make_tuple(has_it, has_type); + }) .def("metadata_num", &Array::metadata_num) - .def("delete_array", - py::overload_cast( - &Array::delete_array)); + .def( + "delete_array", + py::overload_cast( + &Array::delete_array)); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/attribute.cc b/tiledb/cc/attribute.cc index 3179ea3701..97ff6ac589 100644 --- a/tiledb/cc/attribute.cc +++ b/tiledb/cc/attribute.cc @@ -14,13 +14,13 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void set_fill_value(Attribute &attr, py::array value) { +void set_fill_value(Attribute& attr, py::array value) { attr.set_fill_value(value.data(), value.nbytes()); } -py::array get_fill_value(Attribute &attr) { +py::array get_fill_value(Attribute& attr) { // Get the fill value from the C++ API as a void* value. - const void *value; + const void* value; uint64_t size; attr.get_fill_value(&value, &size); @@ -60,30 +60,29 @@ py::array get_fill_value(Attribute &attr) { return py::array(value_type, value_num, value); } -void set_enumeration_name(Attribute &attr, const Context &ctx, - const std::string &enumeration_name) { +void set_enumeration_name( + Attribute& attr, const Context& ctx, const std::string& enumeration_name) { AttributeExperimental::set_enumeration_name(ctx, attr, enumeration_name); } -std::optional get_enumeration_name(Attribute &attr, - const Context &ctx) { +std::optional get_enumeration_name( + Attribute& attr, const Context& ctx) { return AttributeExperimental::get_enumeration_name(ctx, attr); } -void init_attribute(py::module &m) { +void init_attribute(py::module& m) { py::class_(m, "Attribute") .def(py::init()) - .def(py::init()) + .def(py::init()) - .def( - py::init()) + .def(py::init()) - .def(py::init()) + .def(py::init()) .def( "__capsule__", - [](Attribute &attr) { return py::capsule(attr.ptr().get(), "attr"); }) + [](Attribute& attr) { return py::capsule(attr.ptr().get(), "attr"); }) .def_property_readonly("_name", &Attribute::name) @@ -91,13 +90,13 @@ void init_attribute(py::module &m) { .def_property("_nullable", &Attribute::nullable, &Attribute::set_nullable) - .def_property("_ncell", &Attribute::cell_val_num, - &Attribute::set_cell_val_num) + .def_property( + "_ncell", &Attribute::cell_val_num, &Attribute::set_cell_val_num) .def_property_readonly("_var", &Attribute::variable_sized) - .def_property("_filters", &Attribute::filter_list, - &Attribute::set_filter_list) + .def_property( + "_filters", &Attribute::filter_list, &Attribute::set_filter_list) .def_property_readonly("_cell_size", &Attribute::cell_size) @@ -107,7 +106,7 @@ void init_attribute(py::module &m) { .def("_set_enumeration_name", set_enumeration_name) - .def("_dump", [](Attribute &attr) { attr.dump(); }); + .def("_dump", [](Attribute& attr) { attr.dump(); }); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/common.cc b/tiledb/cc/common.cc index 9a6be60cf7..84fb35a647 100644 --- a/tiledb/cc/common.cc +++ b/tiledb/cc/common.cc @@ -86,19 +86,20 @@ std::unordered_map _np_name_to_tdb_dtype = { namespace tiledbpy::common { -size_t buffer_nbytes(py::buffer_info &info) { - return info.itemsize * std::accumulate(info.shape.begin(), info.shape.end(), - 1, std::multiplies<>()); +size_t buffer_nbytes(py::buffer_info& info) { + return info.itemsize * + std::accumulate( + info.shape.begin(), info.shape.end(), 1, std::multiplies<>()); } -bool expect_buffer_nbytes(py::buffer_info &info, tiledb_datatype_t datatype, - size_t nelem) { +bool expect_buffer_nbytes( + py::buffer_info& info, tiledb_datatype_t datatype, size_t nelem) { size_t nbytes = buffer_nbytes(info); size_t nbytes_expected = tiledb_datatype_size(datatype) * nelem; return nbytes == nbytes_expected; } -} // namespace tiledbpy::common +} // namespace tiledbpy::common py::dtype tdb_to_np_dtype(tiledb_datatype_t type, uint32_t cell_val_num) { if (type == TILEDB_CHAR || type == TILEDB_STRING_UTF8 || @@ -142,9 +143,9 @@ py::dtype tdb_to_np_dtype(tiledb_datatype_t type, uint32_t cell_val_num) { return np_dtype(rec_list); } - TPY_ERROR_LOC("tiledb datatype not understood ('" + - tiledb::impl::type_to_str(type) + - "', cell_val_num: " + std::to_string(cell_val_num) + ")"); + TPY_ERROR_LOC( + "tiledb datatype not understood ('" + tiledb::impl::type_to_str(type) + + "', cell_val_num: " + std::to_string(cell_val_num) + ")"); } tiledb_datatype_t np_to_tdb_dtype(py::dtype type) { @@ -158,36 +159,37 @@ tiledb_datatype_t np_to_tdb_dtype(py::dtype type) { if (kind.is(py::str("U"))) return TILEDB_STRING_UTF8; - TPY_ERROR_LOC("could not handle numpy dtype: " + - py::getattr(type, "name").cast()); + TPY_ERROR_LOC( + "could not handle numpy dtype: " + + py::getattr(type, "name").cast()); } bool is_tdb_num(tiledb_datatype_t type) { switch (type) { - case TILEDB_INT8: - case TILEDB_INT16: - case TILEDB_UINT8: - case TILEDB_INT32: - case TILEDB_INT64: - case TILEDB_UINT16: - case TILEDB_UINT32: - case TILEDB_UINT64: - case TILEDB_FLOAT32: - case TILEDB_FLOAT64: - return true; - default: - return false; + case TILEDB_INT8: + case TILEDB_INT16: + case TILEDB_UINT8: + case TILEDB_INT32: + case TILEDB_INT64: + case TILEDB_UINT16: + case TILEDB_UINT32: + case TILEDB_UINT64: + case TILEDB_FLOAT32: + case TILEDB_FLOAT64: + return true; + default: + return false; } } bool is_tdb_str(tiledb_datatype_t type) { switch (type) { - case TILEDB_STRING_ASCII: - case TILEDB_STRING_UTF8: - case TILEDB_CHAR: - return true; - default: - return false; + case TILEDB_STRING_ASCII: + case TILEDB_STRING_UTF8: + case TILEDB_CHAR: + return true; + default: + return false; } } @@ -197,8 +199,8 @@ py::size_t get_ncells(py::dtype type) { if (type.is(py::dtype("U"))) { auto np_unicode_size = py::dtype("U").itemsize(); - return type.itemsize() == 0 ? TILEDB_VAR_NUM - : type.itemsize() / np_unicode_size; + return type.itemsize() == 0 ? TILEDB_VAR_NUM : + type.itemsize() / np_unicode_size; } auto np = py::module::import("numpy"); @@ -211,8 +213,8 @@ py::size_t get_ncells(py::dtype type) { return 1; } -py::array_t -uint8_bool_to_uint8_bitmap(py::array_t validity_array) { +py::array_t uint8_bool_to_uint8_bitmap( + py::array_t validity_array) { // TODO profile, probably replace; avoid inplace reassignment auto np = py::module::import("numpy"); auto packbits = np.attr("packbits"); diff --git a/tiledb/cc/common.h b/tiledb/cc/common.h index 4c5725ccd6..c2a0239df0 100644 --- a/tiledb/cc/common.h +++ b/tiledb/cc/common.h @@ -1,5 +1,5 @@ -#include // for enums -#include // C++ +#include // for enums +#include // C++ #include #include @@ -10,29 +10,34 @@ using namespace tiledb; namespace py = pybind11; using namespace pybind11::literals; -#define TPY_ERROR_LOC(m) \ - throw TileDBPyError(std::string(m) + " (" + __FILE__ + ":" + \ - std::to_string(__LINE__) + ")"); +#define TPY_ERROR_LOC(m) \ + throw TileDBPyError( \ + std::string(m) + " (" + __FILE__ + ":" + std::to_string(__LINE__) + \ + ")"); class TileDBPyError : std::runtime_error { -public: - explicit TileDBPyError(const char *m) : std::runtime_error(m) {} - explicit TileDBPyError(std::string m) : std::runtime_error(m.c_str()) {} + public: + explicit TileDBPyError(const char* m) + : std::runtime_error(m) { + } + explicit TileDBPyError(std::string m) + : std::runtime_error(m.c_str()) { + } -public: - virtual const char *what() const noexcept override { + public: + virtual const char* what() const noexcept override { return std::runtime_error::what(); } }; namespace tiledbpy::common { -size_t buffer_nbytes(py::buffer_info &info); +size_t buffer_nbytes(py::buffer_info& info); -bool expect_buffer_nbytes(py::buffer_info &info, tiledb_datatype_t datatype, - size_t nbytes); +bool expect_buffer_nbytes( + py::buffer_info& info, tiledb_datatype_t datatype, size_t nbytes); -} // namespace tiledbpy::common +} // namespace tiledbpy::common py::dtype tdb_to_np_dtype(tiledb_datatype_t type, uint32_t cell_val_num); tiledb_datatype_t np_to_tdb_dtype(py::dtype type); @@ -40,4 +45,4 @@ tiledb_datatype_t np_to_tdb_dtype(py::dtype type); bool is_tdb_num(tiledb_datatype_t type); bool is_tdb_str(tiledb_datatype_t type); -py::size_t get_ncells(py::dtype type); \ No newline at end of file +py::size_t get_ncells(py::dtype type); diff --git a/tiledb/cc/consolidation_plan.cc b/tiledb/cc/consolidation_plan.cc index c1dd4c36bb..05c3490892 100644 --- a/tiledb/cc/consolidation_plan.cc +++ b/tiledb/cc/consolidation_plan.cc @@ -14,15 +14,17 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_consolidation_plan(py::module &m) { +void init_consolidation_plan(py::module& m) { py::class_(m, "ConsolidationPlan") - .def(py::init(), py::keep_alive<1, 2>(), - py::keep_alive<1, 3>()) + .def( + py::init(), + py::keep_alive<1, 2>(), + py::keep_alive<1, 3>()) .def_property_readonly("_num_nodes", &ConsolidationPlan::num_nodes) .def("_num_fragments", &ConsolidationPlan::num_fragments) .def("_fragment_uri", &ConsolidationPlan::fragment_uri) .def("_dump", &ConsolidationPlan::dump); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/context.cc b/tiledb/cc/context.cc index 45a06790fb..48c5a83f44 100644 --- a/tiledb/cc/context.cc +++ b/tiledb/cc/context.cc @@ -1,4 +1,4 @@ -#include // C++ +#include // C++ #include #include @@ -10,18 +10,20 @@ namespace libtiledbcpp { using namespace tiledb; namespace py = pybind11; -void init_context(py::module &m) { +void init_context(py::module& m) { py::class_(m, "Context") .def(py::init()) .def(py::init()) .def(py::init()) .def(py::init()) - .def("__capsule__", - [](Context &ctx) { return py::capsule(ctx.ptr().get(), "ctx"); }) + .def( + "__capsule__", + [](Context& ctx) { return py::capsule(ctx.ptr().get(), "ctx"); }) - .def("__capsule__", - [](Context &ctx) { return py::capsule(ctx.ptr().get(), "ctx"); }) + .def( + "__capsule__", + [](Context& ctx) { return py::capsule(ctx.ptr().get(), "ctx"); }) .def("config", &Context::config) .def("set_tag", &Context::set_tag) @@ -29,55 +31,63 @@ void init_context(py::module &m) { .def("is_supported_fs", &Context::is_supported_fs); } -void init_config(py::module &m) { +void init_config(py::module& m) { py::class_(m, "Config") .def(py::init()) .def(py::init()) .def(py::init>()) .def(py::init()) - .def("__capsule__", - [](Config &config) { - return py::capsule(config.ptr().get(), "config"); - }) + .def( + "__capsule__", + [](Config& config) { + return py::capsule(config.ptr().get(), "config"); + }) .def("set", &Config::set) .def("get", &Config::get) - .def("update", - [](Config &cfg, py::dict &odict) { - for (auto item : odict) { - cfg.set(item.first.cast(), item.second.cast()); - } - }) + .def( + "update", + [](Config& cfg, py::dict& odict) { + for (auto item : odict) { + cfg.set(item.first.cast(), item.second.cast()); + } + }) .def("save_to_file", &Config::save_to_file) .def("__eq__", &Config::operator==) .def("__ne__", &Config::operator!=) //.def("_ptr", &Config::ptr) // TBD should this be capsule? - .def("__setitem__", [](Config &cfg, std::string ¶m, - std::string &val) { cfg[param] = val; }) - .def("__getitem__", - [](const Config &cfg, std::string ¶m) { - try { - return cfg.get(param); - } catch (TileDBError &e) { - throw py::key_error(); - } - }) - .def("__delitem__", - [](Config &cfg, const std::string ¶m) { - try { - cfg.unset(param); - } catch (TileDBError &e) { - throw py::key_error(); - } - }) + .def( + "__setitem__", + [](Config& cfg, std::string& param, std::string& val) { + cfg[param] = val; + }) + .def( + "__getitem__", + [](const Config& cfg, std::string& param) { + try { + return cfg.get(param); + } catch (TileDBError& e) { + throw py::key_error(); + } + }) + .def( + "__delitem__", + [](Config& cfg, const std::string& param) { + try { + cfg.unset(param); + } catch (TileDBError& e) { + throw py::key_error(); + } + }) .def( "_iter", - [](Config &cfg, std::string prefix) { + [](Config& cfg, std::string prefix) { return py::make_iterator(cfg.begin(prefix), cfg.end()); }, - py::keep_alive<0, 1>(), py::arg("prefix") = "") + py::keep_alive<0, 1>(), + py::arg("prefix") = "") .def("unset", &Config::unset); } -}; // namespace libtiledbcpp +}; // namespace libtiledbcpp diff --git a/tiledb/cc/current_domain.cc b/tiledb/cc/current_domain.cc index f45b1c33f4..8374b0aaef 100644 --- a/tiledb/cc/current_domain.cc +++ b/tiledb/cc/current_domain.cc @@ -15,205 +15,282 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_current_domain(py::module &m) { +void init_current_domain(py::module& m) { #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25 py::class_(m, "NDRectangle") .def(py::init()) - .def(py::init()) - - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def( - "_set_range", - [](NDRectangle &ndrect, const std::string &dim_name, - const std::string &start, const std::string &end) { + .def(py::init()) + + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + [](NDRectangle& ndrect, + const std::string& dim_name, + const std::string& start, + const std::string& end) { return ndrect.set_range(dim_name, start, end); }, - py::arg("dim_name"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def("_set_range", - py::overload_cast( - &NDRectangle::set_range), - py::arg("dim_idx"), py::arg("start"), py::arg("end")) - .def( - "_set_range", - [](NDRectangle &ndrect, uint32_t dim_idx, const std::string &start, - const std::string &end) { + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + py::overload_cast( + &NDRectangle::set_range), + py::arg("dim_idx"), + py::arg("start"), + py::arg("end")) + .def( + "_set_range", + [](NDRectangle& ndrect, + uint32_t dim_idx, + const std::string& start, + const std::string& end) { return ndrect.set_range(dim_idx, start, end); }, - py::arg("dim_name"), py::arg("start"), py::arg("end")) - - .def("_range", - [](NDRectangle &ndrect, const std::string &dim_name, - const py::dtype &n_type) -> py::tuple { - if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else if (py::getattr(n_type, "kind").is(py::str("S")) || - py::getattr(n_type, "kind").is(py::str("U"))) { - auto range = ndrect.range(dim_name); - return py::make_tuple(range[0], range[1]); - } else { - TPY_ERROR_LOC("Unsupported type for NDRectangle's range"); - } - }) - .def("_range", - [](NDRectangle &ndrect, unsigned dim_idx, - const py::dtype &n_type) -> py::tuple { - if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (n_type.is(py::dtype::of())) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else if (py::getattr(n_type, "kind").is(py::str("S")) || - py::getattr(n_type, "kind").is(py::str("U"))) { - auto range = ndrect.range(dim_idx); - return py::make_tuple(range[0], range[1]); - } else { - TPY_ERROR_LOC("Unsupported type for NDRectangle's range"); - } - }); + py::arg("dim_name"), + py::arg("start"), + py::arg("end")) + + .def( + "_range", + [](NDRectangle& ndrect, + const std::string& dim_name, + const py::dtype& n_type) -> py::tuple { + if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else if ( + py::getattr(n_type, "kind").is(py::str("S")) || + py::getattr(n_type, "kind").is(py::str("U"))) { + auto range = ndrect.range(dim_name); + return py::make_tuple(range[0], range[1]); + } else { + TPY_ERROR_LOC("Unsupported type for NDRectangle's range"); + } + }) + .def( + "_range", + [](NDRectangle& ndrect, + unsigned dim_idx, + const py::dtype& n_type) -> py::tuple { + if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if (n_type.is(py::dtype::of())) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else if ( + py::getattr(n_type, "kind").is(py::str("S")) || + py::getattr(n_type, "kind").is(py::str("U"))) { + auto range = ndrect.range(dim_idx); + return py::make_tuple(range[0], range[1]); + } else { + TPY_ERROR_LOC("Unsupported type for NDRectangle's range"); + } + }); py::class_(m, "CurrentDomain") .def(py::init()) - .def(py::init()) + .def(py::init()) - .def("__capsule__", - [](CurrentDomain &curr_dom) { - return py::capsule(curr_dom.ptr().get(), "curr_dom"); - }) + .def( + "__capsule__", + [](CurrentDomain& curr_dom) { + return py::capsule(curr_dom.ptr().get(), "curr_dom"); + }) .def_property_readonly("_type", &CurrentDomain::type) - .def("_set_ndrectangle", &CurrentDomain::set_ndrectangle, - py::arg("ndrect")) + .def( + "_set_ndrectangle", + &CurrentDomain::set_ndrectangle, + py::arg("ndrect")) .def("_ndrectangle", &CurrentDomain::ndrectangle) @@ -221,4 +298,4 @@ void init_current_domain(py::module &m) { #endif } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/dimension_label.cc b/tiledb/cc/dimension_label.cc index 4628517ede..ac6684755f 100644 --- a/tiledb/cc/dimension_label.cc +++ b/tiledb/cc/dimension_label.cc @@ -14,33 +14,34 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_dimension_label(py::module &m) { +void init_dimension_label(py::module& m) { py::class_(m, "DimensionLabel") .def(py::init()) - .def(py::init()) + .def(py::init()) - .def("__capsule__", - [](DimensionLabel &dim_label) { - return py::capsule(dim_label.ptr().get(), "dim_label"); - }) + .def( + "__capsule__", + [](DimensionLabel& dim_label) { + return py::capsule(dim_label.ptr().get(), "dim_label"); + }) - .def_property_readonly("_label_attr_name", - &DimensionLabel::label_attr_name) + .def_property_readonly( + "_label_attr_name", &DimensionLabel::label_attr_name) .def_property_readonly("_dim_index", &DimensionLabel::dimension_index) - .def_property_readonly("_tiledb_label_order", - &DimensionLabel::label_order) + .def_property_readonly( + "_tiledb_label_order", &DimensionLabel::label_order) .def_property_readonly("_tiledb_label_dtype", &DimensionLabel::label_type) - .def_property_readonly("_label_ncell", - &DimensionLabel::label_cell_val_num) + .def_property_readonly( + "_label_ncell", &DimensionLabel::label_cell_val_num) .def_property_readonly("_name", &DimensionLabel::name) .def_property_readonly("_uri", &DimensionLabel::uri); } -}; // namespace libtiledbcpp +}; // namespace libtiledbcpp diff --git a/tiledb/cc/domain.cc b/tiledb/cc/domain.cc index 8cb777c248..e09b316a97 100644 --- a/tiledb/cc/domain.cc +++ b/tiledb/cc/domain.cc @@ -13,162 +13,165 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_domain(py::module &m) { +void init_domain(py::module& m) { py::class_(m, "Dimension") .def(py::init()) - .def(py::init([](const Context &ctx, const std::string &name, - tiledb_datatype_t datatype, py::object domain, - py::object tile_extent) { - void *dim_dom = nullptr; - void *dim_tile = nullptr; + .def( + py::init([](const Context& ctx, + const std::string& name, + tiledb_datatype_t datatype, + py::object domain, + py::object tile_extent) { + void* dim_dom = nullptr; + void* dim_tile = nullptr; - if (!domain.is_none()) { - py::buffer domain_buffer = py::buffer(domain); - py::buffer_info domain_info = domain_buffer.request(); - dim_dom = domain_info.ptr; - } + if (!domain.is_none()) { + py::buffer domain_buffer = py::buffer(domain); + py::buffer_info domain_info = domain_buffer.request(); + dim_dom = domain_info.ptr; + } - if (!tile_extent.is_none()) { - py::buffer tile_buffer = py::buffer(tile_extent); - py::buffer_info tile_extent_info = tile_buffer.request(); - dim_tile = tile_extent_info.ptr; - } + if (!tile_extent.is_none()) { + py::buffer tile_buffer = py::buffer(tile_extent); + py::buffer_info tile_extent_info = tile_buffer.request(); + dim_tile = tile_extent_info.ptr; + } - return std::make_unique( - Dimension::create(ctx, name, datatype, dim_dom, dim_tile)); - }), - py::keep_alive<1, 2>()) + return std::make_unique( + Dimension::create(ctx, name, datatype, dim_dom, dim_tile)); + }), + py::keep_alive<1, 2>()) - .def(py::init(), py::keep_alive<1, 2>()) + .def(py::init(), py::keep_alive<1, 2>()) .def_property_readonly("_name", &Dimension::name) .def_property_readonly( "_domain", - [](Dimension &dim) { + [](Dimension& dim) { switch (dim.type()) { - case TILEDB_UINT64: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_SEC: - case TILEDB_DATETIME_MS: - case TILEDB_DATETIME_US: - case TILEDB_DATETIME_NS: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: - case TILEDB_INT64: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_UINT32: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_INT32: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_UINT16: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_INT16: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_UINT8: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_INT8: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_FLOAT64: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_FLOAT32: { - auto dom = dim.domain(); - return py::make_tuple(dom.first, dom.second); - } - case TILEDB_STRING_ASCII: { - return py::make_tuple("", ""); - } - default: - TPY_ERROR_LOC("Unsupported dtype for Dimension's domain"); + case TILEDB_UINT64: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_SEC: + case TILEDB_DATETIME_MS: + case TILEDB_DATETIME_US: + case TILEDB_DATETIME_NS: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: + case TILEDB_INT64: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_UINT32: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_INT32: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_UINT16: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_INT16: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_UINT8: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_INT8: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_FLOAT64: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_FLOAT32: { + auto dom = dim.domain(); + return py::make_tuple(dom.first, dom.second); + } + case TILEDB_STRING_ASCII: { + return py::make_tuple("", ""); + } + default: + TPY_ERROR_LOC("Unsupported dtype for Dimension's domain"); } }) .def_property_readonly( "_tile", - [](Dimension &dim) -> py::object { + [](Dimension& dim) -> py::object { switch (dim.type()) { - case TILEDB_UINT64: { - return py::cast(dim.tile_extent()); - } - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_SEC: - case TILEDB_DATETIME_MS: - case TILEDB_DATETIME_US: - case TILEDB_DATETIME_NS: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: - case TILEDB_INT64: { - return py::cast(dim.tile_extent()); - } - case TILEDB_UINT32: { - return py::cast(dim.tile_extent()); - } - case TILEDB_INT32: { - return py::cast(dim.tile_extent()); - } - case TILEDB_UINT16: { - return py::cast(dim.tile_extent()); - } - case TILEDB_INT16: { - return py::cast(dim.tile_extent()); - } - case TILEDB_UINT8: { - return py::cast(dim.tile_extent()); - } - case TILEDB_INT8: { - return py::cast(dim.tile_extent()); - } - case TILEDB_FLOAT64: { - return py::cast(dim.tile_extent()); - } - case TILEDB_FLOAT32: { - return py::cast(dim.tile_extent()); - } - case TILEDB_STRING_ASCII: { - return py::none(); - } - default: - TPY_ERROR_LOC("Unsupported dtype for Dimension's tile extent"); + case TILEDB_UINT64: { + return py::cast(dim.tile_extent()); + } + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_SEC: + case TILEDB_DATETIME_MS: + case TILEDB_DATETIME_US: + case TILEDB_DATETIME_NS: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: + case TILEDB_INT64: { + return py::cast(dim.tile_extent()); + } + case TILEDB_UINT32: { + return py::cast(dim.tile_extent()); + } + case TILEDB_INT32: { + return py::cast(dim.tile_extent()); + } + case TILEDB_UINT16: { + return py::cast(dim.tile_extent()); + } + case TILEDB_INT16: { + return py::cast(dim.tile_extent()); + } + case TILEDB_UINT8: { + return py::cast(dim.tile_extent()); + } + case TILEDB_INT8: { + return py::cast(dim.tile_extent()); + } + case TILEDB_FLOAT64: { + return py::cast(dim.tile_extent()); + } + case TILEDB_FLOAT32: { + return py::cast(dim.tile_extent()); + } + case TILEDB_STRING_ASCII: { + return py::none(); + } + default: + TPY_ERROR_LOC("Unsupported dtype for Dimension's tile extent"); } }) - .def_property("_filters", &Dimension::filter_list, - &Dimension::set_filter_list) + .def_property( + "_filters", &Dimension::filter_list, &Dimension::set_filter_list) - .def_property("_ncell", &Dimension::cell_val_num, - &Dimension::set_cell_val_num) + .def_property( + "_ncell", &Dimension::cell_val_num, &Dimension::set_cell_val_num) .def_property_readonly("_tiledb_dtype", &Dimension::type) @@ -177,15 +180,16 @@ void init_domain(py::module &m) { py::class_(m, "Domain") .def(py::init()) - .def(py::init()) + .def(py::init()) - .def(py::init()) + .def(py::init()) - .def("__capsule__", - [](Domain &dom) { return py::capsule(dom.ptr().get(), "dom"); }) + .def( + "__capsule__", + [](Domain& dom) { return py::capsule(dom.ptr().get(), "dom"); }) - .def_property_readonly("_ncell", - [](Domain &dom) { return dom.cell_num(); }) + .def_property_readonly( + "_ncell", [](Domain& dom) { return dom.cell_num(); }) .def_property_readonly("_tiledb_dtype", &Domain::type) @@ -194,14 +198,15 @@ void init_domain(py::module &m) { .def_property_readonly("_dims", &Domain::dimensions) .def("_dim", py::overload_cast(&Domain::dimension, py::const_)) - .def("_dim", py::overload_cast(&Domain::dimension, - py::const_)) + .def( + "_dim", + py::overload_cast(&Domain::dimension, py::const_)) .def("_has_dim", &Domain::has_dimension) .def("_add_dim", &Domain::add_dimension, py::keep_alive<1, 2>()) - .def("_dump", [](Domain &dom) { dom.dump(); }); + .def("_dump", [](Domain& dom) { dom.dump(); }); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/enum.cc b/tiledb/cc/enum.cc index d4a88c7afd..01201fe78e 100644 --- a/tiledb/cc/enum.cc +++ b/tiledb/cc/enum.cc @@ -10,7 +10,7 @@ namespace libtiledbcpp { using namespace tiledb; namespace py = pybind11; -void init_enums(py::module &m) { +void init_enums(py::module& m) { // consts from tiledb.h m.def("TILEDB_VAR_NUM", []() { return TILEDB_VAR_NUM; }); m.def("TILEDB_MAX_PATH", []() { return TILEDB_MAX_PATH; }); @@ -64,7 +64,7 @@ void init_enums(py::module &m) { .value("GEOM_WKB", TILEDB_GEOM_WKB) .value("GEOM_WKT", TILEDB_GEOM_WKT) #endif - ; // line continuation for ifdef + ; // line continuation for ifdef py::enum_(m, "ArrayType") .value("DENSE", TILEDB_DENSE) @@ -112,8 +112,9 @@ void init_enums(py::module &m) { .value("WEBP_INPUT_FORMAT", TILEDB_WEBP_INPUT_FORMAT) .value("WEBP_QUALITY", TILEDB_WEBP_QUALITY) .value("WEBP_LOSSLESS", TILEDB_WEBP_LOSSLESS) - .value("COMPRESSION_REINTERPRET_DATATYPE", - TILEDB_COMPRESSION_REINTERPRET_DATATYPE); + .value( + "COMPRESSION_REINTERPRET_DATATYPE", + TILEDB_COMPRESSION_REINTERPRET_DATATYPE); py::enum_(m, "WebpInputFormat") .value("WEBP_NONE", TILEDB_WEBP_NONE) @@ -140,8 +141,8 @@ void init_enums(py::module &m) { .value("DELETE", TILEDB_DELETE) .value("MODIFY_EXCLUSIVE", TILEDB_MODIFY_EXCLUSIVE); - py::enum_(m, "QueryConditionOp", - py::module_local()) + py::enum_( + m, "QueryConditionOp", py::module_local()) .value("LT", TILEDB_LT) .value("LE", TILEDB_LE) .value("GT", TILEDB_GT) @@ -177,12 +178,15 @@ void init_enums(py::module &m) { #endif // test helpers to check enum name against typed value m.def("_enum_string", &tiledb::impl::type_to_str); - m.def("_enum_string", - py::overload_cast(&tiledb::ArraySchema::to_str)); - m.def("_enum_string", - py::overload_cast(&tiledb::ArraySchema::to_str)); - m.def("_enum_string", - py::overload_cast(&tiledb::Filter::to_str)); + m.def( + "_enum_string", + py::overload_cast(&tiledb::ArraySchema::to_str)); + m.def( + "_enum_string", + py::overload_cast(&tiledb::ArraySchema::to_str)); + m.def( + "_enum_string", + py::overload_cast(&tiledb::Filter::to_str)); m.def("_enum_string", [](Query::Status status) { std::stringstream ss; ss << status; @@ -190,4 +194,4 @@ void init_enums(py::module &m) { }); } -}; // namespace libtiledbcpp +}; // namespace libtiledbcpp diff --git a/tiledb/cc/enumeration.cc b/tiledb/cc/enumeration.cc index 131de3a533..3e240c75ef 100644 --- a/tiledb/cc/enumeration.cc +++ b/tiledb/cc/enumeration.cc @@ -14,59 +14,74 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_enumeration(py::module &m) { +void init_enumeration(py::module& m) { py::class_(m, "Enumeration") .def(py::init()) - .def(py::init([](const Context &ctx, const std::string &name, - py::dtype type, bool ordered) { + .def(py::init([](const Context& ctx, + const std::string& name, + py::dtype type, + bool ordered) { tiledb_datatype_t data_type; try { data_type = np_to_tdb_dtype(type); - } catch (const TileDBPyError &e) { + } catch (const TileDBPyError& e) { throw py::type_error(e.what()); } py::size_t cell_val_num = get_ncells(type); - return Enumeration::create_empty(ctx, name, data_type, cell_val_num, - ordered); + return Enumeration::create_empty( + ctx, name, data_type, cell_val_num, ordered); })) - .def(py::init([](const Context &ctx, const std::string &name, - std::vector &values, bool ordered, + .def(py::init([](const Context& ctx, + const std::string& name, + std::vector& values, + bool ordered, tiledb_datatype_t type) { return Enumeration::create(ctx, name, values, ordered, type); })) - .def(py::init([](const Context &ctx, const std::string &name, - bool ordered, py::array data, py::array offsets) { + .def(py::init([](const Context& ctx, + const std::string& name, + bool ordered, + py::array data, + py::array offsets) { tiledb_datatype_t data_type; try { data_type = np_to_tdb_dtype(data.dtype()); - } catch (const TileDBPyError &e) { + } catch (const TileDBPyError& e) { throw py::type_error(e.what()); } py::buffer_info data_buffer = data.request(); if (data_buffer.ndim != 1) - throw py::type_error("Only 1D Numpy arrays can be stored as " - "enumeration values"); + throw py::type_error( + "Only 1D Numpy arrays can be stored as " + "enumeration values"); py::size_t cell_val_num = offsets.size() == 0 ? get_ncells(data.dtype()) : TILEDB_VAR_NUM; return Enumeration::create( - ctx, name, data_type, cell_val_num, ordered, data.data(), - data.nbytes(), offsets.size() == 0 ? nullptr : offsets.data(), + ctx, + name, + data_type, + cell_val_num, + ordered, + data.data(), + data.nbytes(), + offsets.size() == 0 ? nullptr : offsets.data(), offsets.nbytes()); })) - .def(py::init(), py::keep_alive<1, 2>()) + .def(py::init(), py::keep_alive<1, 2>()) - .def("__capsule__", - [](Enumeration &enmr) { - return py::capsule(enmr.ptr().get(), "enmr"); - }) + .def( + "__capsule__", + [](Enumeration& enmr) { + return py::capsule(enmr.ptr().get(), "enmr"); + }) .def_property_readonly("name", &Enumeration::name) @@ -76,23 +91,25 @@ void init_enumeration(py::module &m) { .def_property_readonly("ordered", &Enumeration::ordered) - .def("values", - [](Enumeration &enmr) { - auto data = enmr.as_vector(); - auto dtype = tdb_to_np_dtype(enmr.type(), enmr.cell_val_num()); - return py::array(dtype, data.size() / dtype.itemsize(), - data.data()); - }) - .def("str_values", - [](Enumeration &enmr) { return enmr.as_vector(); }) + .def( + "values", + [](Enumeration& enmr) { + auto data = enmr.as_vector(); + auto dtype = tdb_to_np_dtype(enmr.type(), enmr.cell_val_num()); + return py::array( + dtype, data.size() / dtype.itemsize(), data.data()); + }) + .def( + "str_values", + [](Enumeration& enmr) { return enmr.as_vector(); }) .def( "extend", - static_cast &)>( + static_cast&)>( &Enumeration::extend)) - .def("extend", [](Enumeration &enmr, py::array data) { + .def("extend", [](Enumeration& enmr, py::array data) { return enmr.extend(data.data(), data.nbytes(), nullptr, 0); }); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/filestore.cc b/tiledb/cc/filestore.cc index 72fd5bee61..383c129193 100644 --- a/tiledb/cc/filestore.cc +++ b/tiledb/cc/filestore.cc @@ -15,40 +15,52 @@ using namespace tiledbpy::common; namespace py = pybind11; class Filestore { -public: + public: // TODO this works, but isn't actually in use at the moment. // we are still using tiledb.libtiledb.ArraySchema. when we switch to using // tiledb.cc.ArraySchema, use this function instead. - static ArraySchema schema_create(const Context &ctx, const char *uri) { - tiledb_array_schema_t *schema; + static ArraySchema schema_create(const Context& ctx, const char* uri) { + tiledb_array_schema_t* schema; tiledb_filestore_schema_create(ctx.ptr().get(), uri, &schema); return ArraySchema(ctx, py::capsule(schema)); } - static void uri_import(const Context &ctx, const char *filestore_array_uri, - const char *file_uri, tiledb_mime_type_t mime_type) { + static void uri_import( + const Context& ctx, + const char* filestore_array_uri, + const char* file_uri, + tiledb_mime_type_t mime_type) { ctx.handle_error(tiledb_filestore_uri_import( ctx.ptr().get(), filestore_array_uri, file_uri, mime_type)); } - static void uri_export(const Context &ctx, const char *filestore_array_uri, - const char *file_uri) { - ctx.handle_error(tiledb_filestore_uri_export(ctx.ptr().get(), file_uri, - filestore_array_uri)); + static void uri_export( + const Context& ctx, + const char* filestore_array_uri, + const char* file_uri) { + ctx.handle_error(tiledb_filestore_uri_export( + ctx.ptr().get(), file_uri, filestore_array_uri)); } - static void buffer_import(const Context &ctx, const char *filestore_array_uri, - py::buffer buf, tiledb_mime_type_t mime_type) { - + static void buffer_import( + const Context& ctx, + const char* filestore_array_uri, + py::buffer buf, + tiledb_mime_type_t mime_type) { py::buffer_info buffer = buf.request(); - ctx.handle_error( - tiledb_filestore_buffer_import(ctx.ptr().get(), filestore_array_uri, - buffer.ptr, py::len(buf), mime_type)); + ctx.handle_error(tiledb_filestore_buffer_import( + ctx.ptr().get(), + filestore_array_uri, + buffer.ptr, + py::len(buf), + mime_type)); } - static py::bytes buffer_export(const Context &ctx, - const char *filestore_array_uri, size_t offset, - size_t size) { + static py::bytes buffer_export( + const Context& ctx, + const char* filestore_array_uri, + size_t offset, + size_t size) { py::array data = py::array(py::dtype::of(), size); py::buffer_info buffer = data.request(); @@ -61,40 +73,40 @@ class Filestore { return to_bytes(data); } - static size_t size(const Context &ctx, const char *filestore_array_uri) { + static size_t size(const Context& ctx, const char* filestore_array_uri) { size_t size; ctx.handle_error( tiledb_filestore_size(ctx.ptr().get(), filestore_array_uri, &size)); return size; } - static const char *mime_type_to_str(tiledb_mime_type_t mime_type) { - const char *str; + static const char* mime_type_to_str(tiledb_mime_type_t mime_type) { + const char* str; tiledb_mime_type_to_str(mime_type, &str); return str; } - static tiledb_mime_type_t mime_type_from_str(const char *str) { + static tiledb_mime_type_t mime_type_from_str(const char* str) { tiledb_mime_type_t mime_type; tiledb_mime_type_from_str(str, &mime_type); return mime_type; } }; -void init_filestore(py::module &m) { +void init_filestore(py::module& m) { py::class_(m, "Filestore") - .def_static("_schema_create", &Filestore::schema_create, - py::keep_alive<1, 2>()) + .def_static( + "_schema_create", &Filestore::schema_create, py::keep_alive<1, 2>()) .def_static("_uri_import", &Filestore::uri_import, py::keep_alive<1, 2>()) .def_static("_uri_export", &Filestore::uri_export, py::keep_alive<1, 2>()) - .def_static("_buffer_import", &Filestore::buffer_import, - py::keep_alive<1, 2>()) - .def_static("_buffer_export", &Filestore::buffer_export, - py::keep_alive<1, 2>()) + .def_static( + "_buffer_import", &Filestore::buffer_import, py::keep_alive<1, 2>()) + .def_static( + "_buffer_export", &Filestore::buffer_export, py::keep_alive<1, 2>()) .def_static("_size", &Filestore::size, py::keep_alive<1, 2>()) .def_static("_mime_type_to_str", &Filestore::mime_type_to_str) .def_static("_mime_type_from_str", &Filestore::mime_type_from_str); ; }; -}; // namespace libtiledbcpp +}; // namespace libtiledbcpp diff --git a/tiledb/cc/filter.cc b/tiledb/cc/filter.cc index 602ac32505..0c39d0d6f9 100644 --- a/tiledb/cc/filter.cc +++ b/tiledb/cc/filter.cc @@ -14,96 +14,101 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_filter(py::module &m) { +void init_filter(py::module& m) { py::class_(m, "Filter") - .def(py::init()) + .def(py::init()) .def_property_readonly("_type", &Filter::filter_type) - .def("_set_option", - [](Filter &filter, Context ctx, tiledb_filter_option_t option, - py::object value) { - switch (option) { - case TILEDB_COMPRESSION_LEVEL: - filter.set_option(option, value.cast()); - break; - case TILEDB_BIT_WIDTH_MAX_WINDOW: - case TILEDB_POSITIVE_DELTA_MAX_WINDOW: - filter.set_option(option, value.cast()); - break; - case TILEDB_SCALE_FLOAT_BYTEWIDTH: - filter.set_option(option, value.cast()); - break; - case TILEDB_SCALE_FLOAT_FACTOR: - case TILEDB_SCALE_FLOAT_OFFSET: - filter.set_option(option, value.cast()); - break; - case TILEDB_WEBP_INPUT_FORMAT: - filter.set_option(option, value.cast()); - break; - case TILEDB_WEBP_QUALITY: - filter.set_option(option, value.cast()); - break; - case TILEDB_WEBP_LOSSLESS: - filter.set_option(option, value.cast()); - break; - case TILEDB_COMPRESSION_REINTERPRET_DATATYPE: - filter.set_option(option, value.cast()); - break; - default: - TPY_ERROR_LOC("Unrecognized filter option to _set_option"); - } - }) + .def( + "_set_option", + [](Filter& filter, + Context ctx, + tiledb_filter_option_t option, + py::object value) { + switch (option) { + case TILEDB_COMPRESSION_LEVEL: + filter.set_option(option, value.cast()); + break; + case TILEDB_BIT_WIDTH_MAX_WINDOW: + case TILEDB_POSITIVE_DELTA_MAX_WINDOW: + filter.set_option(option, value.cast()); + break; + case TILEDB_SCALE_FLOAT_BYTEWIDTH: + filter.set_option(option, value.cast()); + break; + case TILEDB_SCALE_FLOAT_FACTOR: + case TILEDB_SCALE_FLOAT_OFFSET: + filter.set_option(option, value.cast()); + break; + case TILEDB_WEBP_INPUT_FORMAT: + filter.set_option(option, value.cast()); + break; + case TILEDB_WEBP_QUALITY: + filter.set_option(option, value.cast()); + break; + case TILEDB_WEBP_LOSSLESS: + filter.set_option(option, value.cast()); + break; + case TILEDB_COMPRESSION_REINTERPRET_DATATYPE: + filter.set_option(option, value.cast()); + break; + default: + TPY_ERROR_LOC("Unrecognized filter option to _set_option"); + } + }) - .def("_get_option", - [](Filter &filter, Context ctx, - tiledb_filter_option_t option) -> py::object { - switch (option) { - case TILEDB_COMPRESSION_LEVEL: { - int32_t value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_BIT_WIDTH_MAX_WINDOW: - case TILEDB_POSITIVE_DELTA_MAX_WINDOW: { - uint32_t value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_SCALE_FLOAT_BYTEWIDTH: { - uint64_t value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_SCALE_FLOAT_FACTOR: - case TILEDB_SCALE_FLOAT_OFFSET: { - double value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_WEBP_INPUT_FORMAT: { - uint8_t value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_WEBP_QUALITY: { - float value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_WEBP_LOSSLESS: { - uint8_t value; - filter.get_option(option, &value); - return py::cast(value); - } - case TILEDB_COMPRESSION_REINTERPRET_DATATYPE: { - auto value = filter.get_option(option); - return py::cast(static_cast(value)); - } - default: - TPY_ERROR_LOC("Unrecognized filter option to _get_option"); - } - }); + .def( + "_get_option", + [](Filter& filter, + Context ctx, + tiledb_filter_option_t option) -> py::object { + switch (option) { + case TILEDB_COMPRESSION_LEVEL: { + int32_t value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_BIT_WIDTH_MAX_WINDOW: + case TILEDB_POSITIVE_DELTA_MAX_WINDOW: { + uint32_t value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_SCALE_FLOAT_BYTEWIDTH: { + uint64_t value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_SCALE_FLOAT_FACTOR: + case TILEDB_SCALE_FLOAT_OFFSET: { + double value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_WEBP_INPUT_FORMAT: { + uint8_t value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_WEBP_QUALITY: { + float value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_WEBP_LOSSLESS: { + uint8_t value; + filter.get_option(option, &value); + return py::cast(value); + } + case TILEDB_COMPRESSION_REINTERPRET_DATATYPE: { + auto value = filter.get_option(option); + return py::cast(static_cast(value)); + } + default: + TPY_ERROR_LOC("Unrecognized filter option to _get_option"); + } + }); /* needs core patch */ /* .def("_dump", [](Filter &filter, Context ctx) { @@ -117,20 +122,23 @@ void init_filter(py::module &m) { py::class_(m, "FilterList") .def(py::init()) - .def(py::init()) - .def(py::init()) + .def(py::init()) + .def(py::init()) - .def("__capsule__", - [](FilterList &filterlist) { - return py::capsule(filterlist.ptr().get(), "fl"); - }) + .def( + "__capsule__", + [](FilterList& filterlist) { + return py::capsule(filterlist.ptr().get(), "fl"); + }) - .def_property("_chunksize", &FilterList::max_chunk_size, - &FilterList::set_max_chunk_size) + .def_property( + "_chunksize", + &FilterList::max_chunk_size, + &FilterList::set_max_chunk_size) .def("_nfilters", &FilterList::nfilters) .def("_filter", &FilterList::filter) .def("_add_filter", &FilterList::add_filter); -} // namespace libtiledbcpp +} // namespace libtiledbcpp -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/group.cc b/tiledb/cc/group.cc index 3e9a4829e6..4701e87136 100644 --- a/tiledb/cc/group.cc +++ b/tiledb/cc/group.cc @@ -14,11 +14,11 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void put_metadata_numpy(Group &group, const std::string &key, py::array value) { +void put_metadata_numpy(Group& group, const std::string& key, py::array value) { tiledb_datatype_t value_type; try { value_type = np_to_tdb_dtype(value.dtype()); - } catch (const TileDBPyError &e) { + } catch (const TileDBPyError& e) { throw py::type_error(e.what()); } @@ -34,36 +34,39 @@ void put_metadata_numpy(Group &group, const std::string &key, py::array value) { throw py::type_error("Unsupported dtype for metadata"); auto value_num = is_tdb_str(value_type) ? value.nbytes() : value.size(); - group.put_metadata(key, value_type, value_num, - value_num > 0 ? value.data() : nullptr); + group.put_metadata( + key, value_type, value_num, value_num > 0 ? value.data() : nullptr); } -void put_metadata(Group &group, const std::string &key, - tiledb_datatype_t value_type, uint32_t value_num, - const char *value) { +void put_metadata( + Group& group, + const std::string& key, + tiledb_datatype_t value_type, + uint32_t value_num, + const char* value) { group.put_metadata(key, value_type, value_num, value); } -bool has_metadata(Group &group, const std::string &key) { +bool has_metadata(Group& group, const std::string& key) { tiledb_datatype_t _unused_value_type; return group.has_metadata(key, &_unused_value_type); } -std::string get_key_from_index(Group &group, uint64_t index) { +std::string get_key_from_index(Group& group, uint64_t index) { std::string key; tiledb_datatype_t tdb_type; uint32_t value_num; - const void *value; + const void* value; group.get_metadata_from_index(index, &key, &tdb_type, &value_num, &value); return key; } -py::tuple get_metadata(Group &group, const std::string &key) { +py::tuple get_metadata(Group& group, const std::string& key) { tiledb_datatype_t tdb_type; uint32_t value_num; - const void *value; + const void* value; group.get_metadata(key, &tdb_type, &value_num, &value); @@ -84,28 +87,32 @@ py::tuple get_metadata(Group &group, const std::string &key) { return py::make_tuple(py_buf, tdb_type); } -bool has_member(Group &group, std::string obj) { +bool has_member(Group& group, std::string obj) { try { group.member(obj); - } catch (const TileDBError &e) { + } catch (const TileDBError& e) { return false; } return true; } -void init_group(py::module &m) { +void init_group(py::module& m) { py::class_(m, "Group") .def( - py::init(), + py::init(), + py::keep_alive<1, 2>()) + .def( + py::init< + const Context&, + const std::string&, + tiledb_query_type_t, + const Config&>(), py::keep_alive<1, 2>()) - .def(py::init(), - py::keep_alive<1, 2>()) .def("_open", &Group::open) .def("_set_config", &Group::set_config) .def("_config", &Group::config) - .def("_close", [](Group &self) { self.close(true); }) + .def("_close", [](Group& self) { self.close(true); }) .def_property_readonly("_isopen", &Group::is_open) .def_property_readonly("_uri", &Group::uri) .def_property_readonly("_query_type", &Group::query_type) @@ -119,26 +126,39 @@ void init_group(py::module &m) { .def("_get_metadata", get_metadata) .def("_get_key_from_index", get_key_from_index) - .def("_add", &Group::add_member, py::arg("uri"), - py::arg("relative") = false, py::arg("name") = std::nullopt) + .def( + "_add", + &Group::add_member, + py::arg("uri"), + py::arg("relative") = false, + py::arg("name") = std::nullopt) .def("_remove", &Group::remove_member) .def("_delete_group", &Group::delete_group) .def("_member_count", &Group::member_count) - .def("_member", - static_cast(&Group::member)) - .def("_member", - static_cast(&Group::member)) + .def( + "_member", + static_cast(&Group::member)) + .def( + "_member", + static_cast(&Group::member)) .def("_has_member", has_member) .def("_is_relative", &Group::is_relative) .def("_dump", &Group::dump) /* static methods */ .def_static("_create", &Group::create) - .def_static("_consolidate_metadata", &Group::consolidate_metadata, - py::arg("ctx"), py::arg("uri"), - py::arg("config") = (Config *)nullptr) - .def_static("_vacuum_metadata", &Group::vacuum_metadata, py::arg("ctx"), - py::arg("uri"), py::arg("config") = (Config *)nullptr); + .def_static( + "_consolidate_metadata", + &Group::consolidate_metadata, + py::arg("ctx"), + py::arg("uri"), + py::arg("config") = (Config*)nullptr) + .def_static( + "_vacuum_metadata", + &Group::vacuum_metadata, + py::arg("ctx"), + py::arg("uri"), + py::arg("config") = (Config*)nullptr); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/object.cc b/tiledb/cc/object.cc index e0111a1f88..233c8cb164 100644 --- a/tiledb/cc/object.cc +++ b/tiledb/cc/object.cc @@ -13,12 +13,16 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_object(py::module &m) { +void init_object(py::module& m) { py::class_(m, "Object") - .def(py::init &>()) - .def(py::init &>()) + .def(py::init< + const Object::Type&, + const std::string&, + const std::optional&>()) + .def(py::init< + tiledb_object_t, + const std::string&, + const std::optional&>()) .def_property_readonly("_type", &Object::type) .def_property_readonly("_uri", &Object::uri) @@ -30,4 +34,4 @@ void init_object(py::module &m) { .def_static("_move", &Object::move); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/query.cc b/tiledb/cc/query.cc index 145277c33f..b888fb0a1d 100644 --- a/tiledb/cc/query.cc +++ b/tiledb/cc/query.cc @@ -1,4 +1,4 @@ -#include // C++ +#include // C++ #include #include "common.h" @@ -17,19 +17,21 @@ using namespace tiledb; using namespace std; namespace py = pybind11; -void init_query(py::module &m) { +void init_query(py::module& m) { py::class_(m, "Query") //.def(py::init()) - .def(py::init(), - py::keep_alive<1, 2>() /* Keep context alive. */, - py::keep_alive<1, 3>() /* Keep array alive. */) + .def( + py::init(), + py::keep_alive<1, 2>() /* Keep context alive. */, + py::keep_alive<1, 3>() /* Keep array alive. */) - .def(py::init(), - py::keep_alive<1, 2>() /* Keep context alive. */, - py::keep_alive<1, 3>() /* Keep array alive. */) + .def( + py::init(), + py::keep_alive<1, 2>() /* Keep context alive. */, + py::keep_alive<1, 3>() /* Keep array alive. */) // TODO .def("ptr", [&]() -> py::capsule) @@ -37,24 +39,26 @@ void init_query(py::module &m) { .def_property_readonly("query_type", &Query::query_type) - .def_property_readonly("_subarray", - [](Query &query) { - // TODO: Before merge make sure the lifetime of - // the resulting subarray is not tied to this - // query. - Subarray subarray(query.ctx(), query.array()); - query.update_subarray_from_query(&subarray); - return subarray; - }) + .def_property_readonly( + "_subarray", + [](Query& query) { + // TODO: Before merge make sure the lifetime of + // the resulting subarray is not tied to this + // query. + Subarray subarray(query.ctx(), query.array()); + query.update_subarray_from_query(&subarray); + return subarray; + }) // TODO .def("array") -> Array& .def("has_results", &Query::has_results) - .def("is_complete", - [](const Query &query) { - return query.query_status() == Query::Status::COMPLETE; - }) + .def( + "is_complete", + [](const Query& query) { + return query.query_status() == Query::Status::COMPLETE; + }) .def("finalize", &Query::finalize) @@ -70,31 +74,35 @@ void init_query(py::module &m) { // (Query& (Query::*)(const std::string&, void*, // uint64_t))&Query::set_data_buffer); - .def("set_data_buffer", - [](Query &q, std::string name, py::array a) { - // TODO check_type(a.dtype) - // size_t item_size = a.itemsize(); - q.set_data_buffer(name, const_cast(a.data()), a.size()); - }) - - .def("set_offsets_buffer", - [](Query &q, std::string name, py::array a) { - // TODO check_type(a.dtype) - // size_t item_size = a.itemsize(); - q.set_offsets_buffer(name, (uint64_t *)(a.data()), a.size()); - }) - - .def("set_subarray", - [](Query &query, const Subarray &subarray) { - return query.set_subarray(subarray); - }) - - .def("set_validity_buffer", - [](Query &q, std::string name, py::array a) { - // TODO check_type(a.dtype) - // size_t item_size = a.itemsize(); - q.set_validity_buffer(name, (uint8_t *)(a.data()), a.size()); - }) + .def( + "set_data_buffer", + [](Query& q, std::string name, py::array a) { + // TODO check_type(a.dtype) + // size_t item_size = a.itemsize(); + q.set_data_buffer(name, const_cast(a.data()), a.size()); + }) + + .def( + "set_offsets_buffer", + [](Query& q, std::string name, py::array a) { + // TODO check_type(a.dtype) + // size_t item_size = a.itemsize(); + q.set_offsets_buffer(name, (uint64_t*)(a.data()), a.size()); + }) + + .def( + "set_subarray", + [](Query& query, const Subarray& subarray) { + return query.set_subarray(subarray); + }) + + .def( + "set_validity_buffer", + [](Query& q, std::string name, py::array a) { + // TODO check_type(a.dtype) + // size_t item_size = a.itemsize(); + q.set_validity_buffer(name, (uint8_t*)(a.data()), a.size()); + }) .def("submit", &Query::submit, py::call_guard()) @@ -109,4 +117,4 @@ void init_query(py::module &m) { ; } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/schema.cc b/tiledb/cc/schema.cc index 8a85b3d3a0..2010958c1e 100644 --- a/tiledb/cc/schema.cc +++ b/tiledb/cc/schema.cc @@ -1,5 +1,5 @@ -#include // C++ -#include // (needed for dimension labels) +#include // C++ +#include // (needed for dimension labels) #include #include @@ -12,26 +12,37 @@ using namespace tiledb; namespace py = pybind11; class DimensionLabelSchema { -public: - DimensionLabelSchema(tiledb_datatype_t dim_type, py::object dim_tile_extent, - tiledb_data_order_t label_order, - tiledb_datatype_t label_type) - : dim_type_{dim_type}, dim_tile_extent_{dim_tile_extent}, - label_order_{label_order}, label_type_{label_type}, - label_filters_{std::nullopt} {} - - DimensionLabelSchema(tiledb_datatype_t dim_type, py::object dim_tile_extent, - tiledb_data_order_t label_order, - tiledb_datatype_t label_type, - const FilterList &label_filters) - : dim_type_{dim_type}, dim_tile_extent_{dim_tile_extent}, - label_order_{label_order}, label_type_{label_type}, - label_filters_{label_filters} {} - - tiledb_datatype_t dim_type() const { return dim_type_; } - - const void *dim_tile_extent() const { + public: + DimensionLabelSchema( + tiledb_datatype_t dim_type, + py::object dim_tile_extent, + tiledb_data_order_t label_order, + tiledb_datatype_t label_type) + : dim_type_{dim_type} + , dim_tile_extent_{dim_tile_extent} + , label_order_{label_order} + , label_type_{label_type} + , label_filters_{std::nullopt} { + } + + DimensionLabelSchema( + tiledb_datatype_t dim_type, + py::object dim_tile_extent, + tiledb_data_order_t label_order, + tiledb_datatype_t label_type, + const FilterList& label_filters) + : dim_type_{dim_type} + , dim_tile_extent_{dim_tile_extent} + , label_order_{label_order} + , label_type_{label_type} + , label_filters_{label_filters} { + } + + tiledb_datatype_t dim_type() const { + return dim_type_; + } + const void* dim_tile_extent() const { if (dim_tile_extent_.is_none()) { return nullptr; } @@ -40,19 +51,27 @@ class DimensionLabelSchema { return tile_extent_info.ptr; } - bool has_dim_tile_extent() const { return !dim_tile_extent_.is_none(); } + bool has_dim_tile_extent() const { + return !dim_tile_extent_.is_none(); + } - bool has_label_filters() const { return label_filters_.has_value(); } + bool has_label_filters() const { + return label_filters_.has_value(); + } - tiledb_datatype_t label_type() const { return label_type_; } + tiledb_datatype_t label_type() const { + return label_type_; + } - tiledb_data_order_t label_order() const { return label_order_; } + tiledb_data_order_t label_order() const { + return label_order_; + } - const std::optional &label_filters() const { + const std::optional& label_filters() const { return label_filters_; } -private: + private: tiledb_datatype_t dim_type_; py::object dim_tile_extent_; tiledb_data_order_t label_order_; @@ -60,120 +79,133 @@ class DimensionLabelSchema { std::optional label_filters_; }; -void init_schema(py::module &m) { +void init_schema(py::module& m) { py::class_(m, "DimensionLabelSchema") - .def(py::init(), - py::keep_alive<1, 3>()) - - .def(py::init(), - py::keep_alive<1, 3>()) + .def( + py::init< + tiledb_datatype_t, + py::object, + tiledb_data_order_t, + tiledb_datatype_t>(), + py::keep_alive<1, 3>()) + + .def( + py::init< + tiledb_datatype_t, + py::object, + tiledb_data_order_t, + tiledb_datatype_t, + const FilterList&>(), + py::keep_alive<1, 3>()) .def_property_readonly("_dim_dtype", &DimensionLabelSchema::dim_type) - .def_property_readonly("_has_label_filters", - &DimensionLabelSchema::has_label_filters) + .def_property_readonly( + "_has_label_filters", &DimensionLabelSchema::has_label_filters) .def_property_readonly("_label_dtype", &DimensionLabelSchema::label_type) - .def_property_readonly("_label_filters", - [](DimensionLabelSchema &dim_label_schema) { - return dim_label_schema.label_filters().value(); - }) + .def_property_readonly( + "_label_filters", + [](DimensionLabelSchema& dim_label_schema) { + return dim_label_schema.label_filters().value(); + }) .def_property_readonly("_label_order", &DimensionLabelSchema::label_order) .def_property_readonly( "_dim_tile_extent", - [](DimensionLabelSchema &dim_label_schema) -> py::object { - const void *tile_extent = dim_label_schema.dim_tile_extent(); + [](DimensionLabelSchema& dim_label_schema) -> py::object { + const void* tile_extent = dim_label_schema.dim_tile_extent(); if (tile_extent == nullptr) { return py::none(); } auto dim_type = dim_label_schema.dim_type(); switch (dim_type) { - case TILEDB_UINT64: { - using T = uint64_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_SEC: - case TILEDB_DATETIME_MS: - case TILEDB_DATETIME_US: - case TILEDB_DATETIME_NS: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: - case TILEDB_INT64: { - using T = int64_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_UINT32: { - using T = uint32_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_INT32: { - using T = int32_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_UINT16: { - using T = uint16_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_INT16: { - using T = int16_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_UINT8: { - using T = uint8_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_INT8: { - using T = int8_t; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_FLOAT64: { - using T = double; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_FLOAT32: { - using T = float; - return py::cast(*static_cast(tile_extent)); - } - case TILEDB_STRING_ASCII: { - // This should have already been caught in the check for a mullptr - // above. - throw TileDBError("Setting a tile extent on a string dimension " - "is not supported for dimension labels"); - } - default: - throw TileDBError("Unsupported dtype for dimension tile extent"); + case TILEDB_UINT64: { + using T = uint64_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_SEC: + case TILEDB_DATETIME_MS: + case TILEDB_DATETIME_US: + case TILEDB_DATETIME_NS: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: + case TILEDB_INT64: { + using T = int64_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_UINT32: { + using T = uint32_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_INT32: { + using T = int32_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_UINT16: { + using T = uint16_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_INT16: { + using T = int16_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_UINT8: { + using T = uint8_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_INT8: { + using T = int8_t; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_FLOAT64: { + using T = double; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_FLOAT32: { + using T = float; + return py::cast(*static_cast(tile_extent)); + } + case TILEDB_STRING_ASCII: { + // This should have already been caught in the check for a + // mullptr above. + throw TileDBError( + "Setting a tile extent on a string dimension " + "is not supported for dimension labels"); + } + default: + throw TileDBError( + "Unsupported dtype for dimension tile extent"); } }); py::class_(m, "ArraySchema") .def(py::init()) - .def(py::init(), py::keep_alive<1, 2>()) + .def(py::init(), py::keep_alive<1, 2>()) - .def(py::init()) + .def(py::init()) - .def(py::init()) + .def(py::init()) - .def("__capsule__", - [](ArraySchema &schema) { - return py::capsule(schema.ptr().get(), "schema"); - }) + .def( + "__capsule__", + [](ArraySchema& schema) { + return py::capsule(schema.ptr().get(), "schema"); + }) .def("_dump", &ArraySchema::dump) - .def("_dump", [](ArraySchema &schema) { schema.dump(); }) + .def("_dump", [](ArraySchema& schema) { schema.dump(); }) .def("_ctx", &ArraySchema::context) @@ -184,40 +216,53 @@ void init_schema(py::module &m) { // .def_property_readonly("timestamp_range", // &ArraySchema::timestamp_range) - .def_property("_capacity", &ArraySchema::capacity, - &ArraySchema::set_capacity) + .def_property( + "_capacity", &ArraySchema::capacity, &ArraySchema::set_capacity) .def_property_readonly("_version", &ArraySchema::version) - .def_property("_cell_order", &ArraySchema::cell_order, - &ArraySchema::set_cell_order) - - .def_property("_tile_order", &ArraySchema::tile_order, - &ArraySchema::set_tile_order) - - .def_property("_allows_dups", &ArraySchema::allows_dups, - &ArraySchema::set_allows_dups) - - .def_property("_coords_filters", &ArraySchema::coords_filter_list, - &ArraySchema::set_coords_filter_list) - - .def_property("_offsets_filters", &ArraySchema::offsets_filter_list, - &ArraySchema::set_offsets_filter_list) - - .def_property("_validity_filters", &ArraySchema::validity_filter_list, - &ArraySchema::set_validity_filter_list) - - .def("_attr", py::overload_cast( - &ArraySchema::attribute, py::const_)) - .def("_attr", - py::overload_cast(&ArraySchema::attribute, py::const_)) - - .def("_dim_label", - [](const ArraySchema &schema, const Context &context, - const std::string &name) { - return ArraySchemaExperimental::dimension_label(context, schema, - name); - }) + .def_property( + "_cell_order", &ArraySchema::cell_order, &ArraySchema::set_cell_order) + + .def_property( + "_tile_order", &ArraySchema::tile_order, &ArraySchema::set_tile_order) + + .def_property( + "_allows_dups", + &ArraySchema::allows_dups, + &ArraySchema::set_allows_dups) + + .def_property( + "_coords_filters", + &ArraySchema::coords_filter_list, + &ArraySchema::set_coords_filter_list) + + .def_property( + "_offsets_filters", + &ArraySchema::offsets_filter_list, + &ArraySchema::set_offsets_filter_list) + + .def_property( + "_validity_filters", + &ArraySchema::validity_filter_list, + &ArraySchema::set_validity_filter_list) + + .def( + "_attr", + py::overload_cast( + &ArraySchema::attribute, py::const_)) + .def( + "_attr", + py::overload_cast(&ArraySchema::attribute, py::const_)) + + .def( + "_dim_label", + [](const ArraySchema& schema, + const Context& context, + const std::string& name) { + return ArraySchemaExperimental::dimension_label( + context, schema, name); + }) .def_property_readonly("_nattr", &ArraySchema::attribute_num) @@ -226,64 +271,82 @@ void init_schema(py::module &m) { .def("_add_attr", &ArraySchema::add_attribute) - .def("_add_dim_label", - [](ArraySchema &schema, const Context &ctx, const std::string &name, - uint32_t dim_index, - const DimensionLabelSchema &dim_label_schema) { - // Check dimension datatype. - auto dim_type = schema.domain().dimension(dim_index).type(); - if (dim_label_schema.dim_type() != dim_type) { - throw TileDBError("Cannot add dimension label '" + name + - "'; The dimension datatype does not match the " - "datatype of the dimension in the array."); - } - - // Add dimension label. - ArraySchemaExperimental::add_dimension_label( - ctx, schema, dim_index, name, dim_label_schema.label_order(), - dim_label_schema.label_type(), - dim_label_schema.label_filters()); - - // If dimension tile extent is set, add dimension tile extent. - if (dim_label_schema.has_dim_tile_extent()) { - ctx.handle_error( - tiledb_array_schema_set_dimension_label_tile_extent( - ctx.ptr().get(), schema.ptr().get(), name.c_str(), - dim_type, dim_label_schema.dim_tile_extent())); - } - }) + .def( + "_add_dim_label", + [](ArraySchema& schema, + const Context& ctx, + const std::string& name, + uint32_t dim_index, + const DimensionLabelSchema& dim_label_schema) { + // Check dimension datatype. + auto dim_type = schema.domain().dimension(dim_index).type(); + if (dim_label_schema.dim_type() != dim_type) { + throw TileDBError( + "Cannot add dimension label '" + name + + "'; The dimension datatype does not match the " + "datatype of the dimension in the array."); + } + + // Add dimension label. + ArraySchemaExperimental::add_dimension_label( + ctx, + schema, + dim_index, + name, + dim_label_schema.label_order(), + dim_label_schema.label_type(), + dim_label_schema.label_filters()); + + // If dimension tile extent is set, add dimension tile extent. + if (dim_label_schema.has_dim_tile_extent()) { + ctx.handle_error( + tiledb_array_schema_set_dimension_label_tile_extent( + ctx.ptr().get(), + schema.ptr().get(), + name.c_str(), + dim_type, + dim_label_schema.dim_tile_extent())); + } + }) .def("_check", &ArraySchema::check) .def("_has_attribute", &ArraySchema::has_attribute) - .def("_has_dim_label", - [](const ArraySchema &schema, const Context &ctx, - const std::string &name) { - return ArraySchemaExperimental::has_dimension_label(ctx, schema, - name); - }) - - .def("_add_enumeration", - [](const ArraySchema &schema, const Context &ctx, - const Enumeration &enmr) { - ArraySchemaExperimental::add_enumeration(ctx, schema, enmr); - }) + .def( + "_has_dim_label", + [](const ArraySchema& schema, + const Context& ctx, + const std::string& name) { + return ArraySchemaExperimental::has_dimension_label( + ctx, schema, name); + }) + + .def( + "_add_enumeration", + [](const ArraySchema& schema, + const Context& ctx, + const Enumeration& enmr) { + ArraySchemaExperimental::add_enumeration(ctx, schema, enmr); + }) #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25 - .def("_current_domain", - [](const ArraySchema &schema, const Context &ctx) { - return ArraySchemaExperimental::current_domain(ctx, schema); - }) - - .def("_set_current_domain", - [](ArraySchema &schema, const Context &ctx, - const CurrentDomain ¤t_domain) { - ArraySchemaExperimental::set_current_domain(ctx, schema, - current_domain); - }) + .def( + "_current_domain", + [](const ArraySchema& schema, const Context& ctx) { + return ArraySchemaExperimental::current_domain(ctx, schema); + }) + + .def( + "_set_current_domain", + [](ArraySchema& schema, + const Context& ctx, + const CurrentDomain& current_domain) { + ArraySchemaExperimental::set_current_domain( + ctx, schema, current_domain); + }) #endif ; } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/subarray.cc b/tiledb/cc/subarray.cc index 673993cdba..b2f2534b46 100644 --- a/tiledb/cc/subarray.cc +++ b/tiledb/cc/subarray.cc @@ -1,7 +1,7 @@ -#include // for enums -#include // C++ -#include // for `tiledb_subarray_has_label_range` -#include // C++ +#include // for enums +#include // for `tiledb_subarray_has_label_range` +#include // C++ +#include // C++ #include "common.h" @@ -15,9 +15,9 @@ namespace libtiledbcpp { using namespace tiledb; namespace py = pybind11; -template struct SubarrayDimensionManipulator { - - static void copy(Subarray &subarray, Subarray &original, uint32_t dim_idx) { +template +struct SubarrayDimensionManipulator { + static void copy(Subarray& subarray, Subarray& original, uint32_t dim_idx) { for (uint64_t range_idx{0}; range_idx < original.range_num(dim_idx); ++range_idx) { std::array range = original.range(dim_idx, range_idx); @@ -25,14 +25,15 @@ template struct SubarrayDimensionManipulator { } } - static py::ssize_t length(Subarray &subarray, uint32_t dim_idx) { + static py::ssize_t length(Subarray& subarray, uint32_t dim_idx) { uint64_t length = 0; for (uint64_t range_idx{0}; range_idx < subarray.range_num(dim_idx); ++range_idx) { std::array range = subarray.range(dim_idx, range_idx); if (range[2] != 0 && range[1] != 1) { - throw TileDBPyError("Support for getting the lenght of ranges with a " - "stride is not yet implemented."); + throw TileDBPyError( + "Support for getting the lenght of ranges with a " + "stride is not yet implemented."); } auto range_length = static_cast(range[1] - range[0]); @@ -48,9 +49,9 @@ template struct SubarrayDimensionManipulator { } }; -template <> struct SubarrayDimensionManipulator { - - static void copy(Subarray &subarray, Subarray &original, uint32_t dim_idx) { +template <> +struct SubarrayDimensionManipulator { + static void copy(Subarray& subarray, Subarray& original, uint32_t dim_idx) { for (uint64_t range_idx{0}; range_idx < original.range_num(dim_idx); ++range_idx) { std::array range = original.range(dim_idx, range_idx); @@ -58,13 +59,13 @@ template <> struct SubarrayDimensionManipulator { } } - static uint64_t length(Subarray &, uint32_t) { + static uint64_t length(Subarray&, uint32_t) { throw TileDBPyError( "Getting length of ranges is not supported on string dimensions."); } }; -void add_dim_range(Subarray &subarray, uint32_t dim_idx, py::tuple r) { +void add_dim_range(Subarray& subarray, uint32_t dim_idx, py::tuple r) { if (py::len(r) == 0) return; else if (py::len(r) != 2) @@ -78,75 +79,188 @@ void add_dim_range(Subarray &subarray, uint32_t dim_idx, py::tuple r) { try { switch (tiledb_type) { + case TILEDB_INT32: { + using T = int32_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT64: { + using T = int64_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT8: { + using T = int8_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT8: { + using T = uint8_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT16: { + using T = int16_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT16: { + using T = uint16_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT32: { + using T = uint32_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT64: { + using T = uint64_t; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_FLOAT32: { + using T = float; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_FLOAT64: { + using T = double; + subarray.add_range(dim_idx, r0.cast(), r1.cast()); + break; + } + case TILEDB_STRING_ASCII: + case TILEDB_STRING_UTF8: + case TILEDB_CHAR: { + if (!py::isinstance(r0) != !py::isinstance(r1)) { + TPY_ERROR_LOC( + "internal error: ranges must both be strings or (None, None)"); + } else if ( + !py::isinstance(r0) && !py::isinstance(r1) && + !py::isinstance(r0) && !py::isinstance(r1) && + !py::isinstance(r0) && !py::isinstance(r1)) { + TPY_ERROR_LOC( + "internal error: expected string type for var-length dim!"); + } + + if (!py::isinstance(r0) && !py::isinstance(r0)) + subarray.add_range( + dim_idx, r0.cast(), r1.cast()); + + break; + } + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_SEC: + case TILEDB_DATETIME_MS: + case TILEDB_DATETIME_US: + case TILEDB_DATETIME_NS: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: { + case TILEDB_TIME_HR: + case TILEDB_TIME_MIN: + case TILEDB_TIME_SEC: + case TILEDB_TIME_MS: + case TILEDB_TIME_US: + case TILEDB_TIME_NS: + case TILEDB_TIME_PS: + case TILEDB_TIME_FS: + case TILEDB_TIME_AS: + py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); + auto dt0 = + py::isinstance(r0) ? r0 : r0.attr("astype")(dtype); + auto dt1 = + py::isinstance(r1) ? r1 : r1.attr("astype")(dtype); + + // TODO, this is suboptimal, should define pybind converter + if (py::isinstance(dt0) && py::isinstance(dt1)) { + subarray.add_range( + dim_idx, py::cast(dt0), py::cast(dt1)); + } else { + auto darray = py::array(py::make_tuple(dt0, dt1)); + subarray.add_range( + dim_idx, *(int64_t*)darray.data(0), *(int64_t*)darray.data(1)); + } + + break; + } + default: + TPY_ERROR_LOC("Unknown dim type conversion!"); + } + } catch (py::cast_error& e) { + (void)e; + std::string msg = "Failed to cast dim range '" + (std::string)py::repr(r) + + "' to dim type " + tiledb::impl::type_to_str(tiledb_type); + TPY_ERROR_LOC(msg); + } +} + +void copy_ranges_on_dim( + Subarray& subarray, Subarray original, uint32_t dim_idx) { + auto tiledb_type = + subarray.array().schema().domain().dimension(dim_idx).type(); + + switch (tiledb_type) { case TILEDB_INT32: { using T = int32_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_INT64: { using T = int64_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_INT8: { using T = int8_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_UINT8: { using T = uint8_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_INT16: { using T = int16_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_UINT16: { using T = uint16_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_UINT32: { using T = uint32_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_UINT64: { using T = uint64_t; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_FLOAT32: { using T = float; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_FLOAT64: { using T = double; - subarray.add_range(dim_idx, r0.cast(), r1.cast()); + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_STRING_ASCII: case TILEDB_STRING_UTF8: case TILEDB_CHAR: { - if (!py::isinstance(r0) != !py::isinstance(r1)) { - TPY_ERROR_LOC( - "internal error: ranges must both be strings or (None, None)"); - } else if (!py::isinstance(r0) && - !py::isinstance(r1) && - !py::isinstance(r0) && !py::isinstance(r1) && - !py::isinstance(r0) && - !py::isinstance(r1)) { - TPY_ERROR_LOC( - "internal error: expected string type for var-length dim!"); - } - - if (!py::isinstance(r0) && !py::isinstance(r0)) - subarray.add_range(dim_idx, r0.cast(), - r1.cast()); - + using T = std::string; + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); break; } case TILEDB_DATETIME_YEAR: @@ -162,320 +276,60 @@ void add_dim_range(Subarray &subarray, uint32_t dim_idx, py::tuple r) { case TILEDB_DATETIME_PS: case TILEDB_DATETIME_FS: case TILEDB_DATETIME_AS: { - case TILEDB_TIME_HR: - case TILEDB_TIME_MIN: - case TILEDB_TIME_SEC: - case TILEDB_TIME_MS: - case TILEDB_TIME_US: - case TILEDB_TIME_NS: - case TILEDB_TIME_PS: - case TILEDB_TIME_FS: - case TILEDB_TIME_AS: - py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); - auto dt0 = py::isinstance(r0) ? r0 : r0.attr("astype")(dtype); - auto dt1 = py::isinstance(r1) ? r1 : r1.attr("astype")(dtype); - - // TODO, this is suboptimal, should define pybind converter - if (py::isinstance(dt0) && py::isinstance(dt1)) { - subarray.add_range(dim_idx, py::cast(dt0), - py::cast(dt1)); - } else { - auto darray = py::array(py::make_tuple(dt0, dt1)); - subarray.add_range(dim_idx, *(int64_t *)darray.data(0), - *(int64_t *)darray.data(1)); - } - - break; + case TILEDB_TIME_HR: + case TILEDB_TIME_MIN: + case TILEDB_TIME_SEC: + case TILEDB_TIME_MS: + case TILEDB_TIME_US: + case TILEDB_TIME_NS: + case TILEDB_TIME_PS: + case TILEDB_TIME_FS: + case TILEDB_TIME_AS: + using T = int64_t; + SubarrayDimensionManipulator::copy(subarray, original, dim_idx); + break; } default: TPY_ERROR_LOC("Unknown dim type conversion!"); - } - } catch (py::cast_error &e) { - (void)e; - std::string msg = "Failed to cast dim range '" + (std::string)py::repr(r) + - "' to dim type " + tiledb::impl::type_to_str(tiledb_type); - TPY_ERROR_LOC(msg); } } -void copy_ranges_on_dim(Subarray &subarray, Subarray original, - uint32_t dim_idx) { - +py::ssize_t length_ranges(Subarray& subarray, uint32_t dim_idx) { auto tiledb_type = subarray.array().schema().domain().dimension(dim_idx).type(); switch (tiledb_type) { - case TILEDB_INT32: { - using T = int32_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_INT64: { - using T = int64_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_INT8: { - using T = int8_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_UINT8: { - using T = uint8_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_INT16: { - using T = int16_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_UINT16: { - using T = uint16_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_UINT32: { - using T = uint32_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_UINT64: { - using T = uint64_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_FLOAT32: { - using T = float; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_FLOAT64: { - using T = double; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_STRING_ASCII: - case TILEDB_STRING_UTF8: - case TILEDB_CHAR: { - using T = std::string; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_SEC: - case TILEDB_DATETIME_MS: - case TILEDB_DATETIME_US: - case TILEDB_DATETIME_NS: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: { - case TILEDB_TIME_HR: - case TILEDB_TIME_MIN: - case TILEDB_TIME_SEC: - case TILEDB_TIME_MS: - case TILEDB_TIME_US: - case TILEDB_TIME_NS: - case TILEDB_TIME_PS: - case TILEDB_TIME_FS: - case TILEDB_TIME_AS: - using T = int64_t; - SubarrayDimensionManipulator::copy(subarray, original, dim_idx); - break; - } - default: - TPY_ERROR_LOC("Unknown dim type conversion!"); - } -} - -py::ssize_t length_ranges(Subarray &subarray, uint32_t dim_idx) { - - auto tiledb_type = - subarray.array().schema().domain().dimension(dim_idx).type(); - - switch (tiledb_type) { - case TILEDB_INT32: { - using T = int32_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_INT64: { - using T = int64_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_INT8: { - using T = int8_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_UINT8: { - using T = uint8_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_INT16: { - using T = int16_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_UINT16: { - using T = uint16_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_UINT32: { - using T = uint32_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_UINT64: { - using T = uint64_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_SEC: - case TILEDB_DATETIME_MS: - case TILEDB_DATETIME_US: - case TILEDB_DATETIME_NS: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: { - case TILEDB_TIME_HR: - case TILEDB_TIME_MIN: - case TILEDB_TIME_SEC: - case TILEDB_TIME_MS: - case TILEDB_TIME_US: - case TILEDB_TIME_NS: - case TILEDB_TIME_PS: - case TILEDB_TIME_FS: - case TILEDB_TIME_AS: - using T = int64_t; - return SubarrayDimensionManipulator::length(subarray, dim_idx); - } - default: - TPY_ERROR_LOC("Dimension length not supported on a dimension with the " - "given datatype."); - } -} - -void add_dim_point_ranges(const Context &ctx, Subarray &subarray, - uint32_t dim_idx, pybind11::handle dim_range) { - - // Cast range object to appropriately typed py::array. - auto tiledb_type = - subarray.array().schema().domain().dimension(dim_idx).type(); - py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); - py::array ranges = dim_range.attr("astype")(dtype); - - // Set point ranges using C-API. - tiledb_ctx_t *c_ctx = ctx.ptr().get(); - tiledb_subarray_t *c_subarray = subarray.ptr().get(); - ctx.handle_error(tiledb_subarray_add_point_ranges( - c_ctx, c_subarray, dim_idx, (void *)ranges.data(), ranges.size())); -} - -void add_label_range(const Context &ctx, Subarray &subarray, - const std::string &label_name, py::tuple r) { - if (py::len(r) == 0) - return; - else if (py::len(r) != 2) - TPY_ERROR_LOC("Unexpected range len != 2"); - - auto r0 = r[0]; - auto r1 = r[1]; - - auto tiledb_type = ArraySchemaExperimental::dimension_label( - ctx, subarray.array().schema(), label_name) - .label_type(); - - try { - switch (tiledb_type) { case TILEDB_INT32: { using T = int32_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_INT64: { using T = int64_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_INT8: { using T = int8_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_UINT8: { using T = uint8_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_INT16: { using T = int16_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_UINT16: { using T = uint16_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_UINT32: { using T = uint32_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_UINT64: { using T = uint64_t; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; - } - case TILEDB_FLOAT32: { - using T = float; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; - } - case TILEDB_FLOAT64: { - using T = double; - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0.cast(), r1.cast()); - break; - } - case TILEDB_STRING_ASCII: - case TILEDB_STRING_UTF8: - case TILEDB_CHAR: { - if (!py::isinstance(r0) != !py::isinstance(r1)) { - TPY_ERROR_LOC( - "internal error: ranges must both be strings or (None, None)"); - } else if (!py::isinstance(r0) && - !py::isinstance(r1) && - !py::isinstance(r0) && !py::isinstance(r1) && - !py::isinstance(r0) && - !py::isinstance(r1)) { - TPY_ERROR_LOC( - "internal error: expected string type for var-length label!"); - } - - if (!py::isinstance(r0) && !py::isinstance(r0)) { - std::string r0_string = r0.cast(); - std::string r1_string = r1.cast(); - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - r0_string, r1_string); - } - break; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } case TILEDB_DATETIME_YEAR: case TILEDB_DATETIME_MONTH: @@ -490,36 +344,195 @@ void add_label_range(const Context &ctx, Subarray &subarray, case TILEDB_DATETIME_PS: case TILEDB_DATETIME_FS: case TILEDB_DATETIME_AS: { - case TILEDB_TIME_HR: - case TILEDB_TIME_MIN: - case TILEDB_TIME_SEC: - case TILEDB_TIME_MS: - case TILEDB_TIME_US: - case TILEDB_TIME_NS: - case TILEDB_TIME_PS: - case TILEDB_TIME_FS: - case TILEDB_TIME_AS: - py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); - auto dt0 = py::isinstance(r0) ? r0 : r0.attr("astype")(dtype); - auto dt1 = py::isinstance(r1) ? r1 : r1.attr("astype")(dtype); - - if (py::isinstance(dt0) && py::isinstance(dt1)) { - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - py::cast(dt0), - py::cast(dt1)); - } else { - auto darray = py::array(py::make_tuple(dt0, dt1)); - SubarrayExperimental::add_label_range(ctx, subarray, label_name, - *(int64_t *)darray.data(0), - *(int64_t *)darray.data(1)); - } - - break; + case TILEDB_TIME_HR: + case TILEDB_TIME_MIN: + case TILEDB_TIME_SEC: + case TILEDB_TIME_MS: + case TILEDB_TIME_US: + case TILEDB_TIME_NS: + case TILEDB_TIME_PS: + case TILEDB_TIME_FS: + case TILEDB_TIME_AS: + using T = int64_t; + return SubarrayDimensionManipulator::length(subarray, dim_idx); } default: - TPY_ERROR_LOC("Unknown dimension label type conversion!"); + TPY_ERROR_LOC( + "Dimension length not supported on a dimension with the " + "given datatype."); + } +} + +void add_dim_point_ranges( + const Context& ctx, + Subarray& subarray, + uint32_t dim_idx, + pybind11::handle dim_range) { + // Cast range object to appropriately typed py::array. + auto tiledb_type = + subarray.array().schema().domain().dimension(dim_idx).type(); + py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); + py::array ranges = dim_range.attr("astype")(dtype); + + // Set point ranges using C-API. + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_subarray_t* c_subarray = subarray.ptr().get(); + ctx.handle_error(tiledb_subarray_add_point_ranges( + c_ctx, c_subarray, dim_idx, (void*)ranges.data(), ranges.size())); +} + +void add_label_range( + const Context& ctx, + Subarray& subarray, + const std::string& label_name, + py::tuple r) { + if (py::len(r) == 0) + return; + else if (py::len(r) != 2) + TPY_ERROR_LOC("Unexpected range len != 2"); + + auto r0 = r[0]; + auto r1 = r[1]; + + auto tiledb_type = ArraySchemaExperimental::dimension_label( + ctx, subarray.array().schema(), label_name) + .label_type(); + + try { + switch (tiledb_type) { + case TILEDB_INT32: { + using T = int32_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT64: { + using T = int64_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT8: { + using T = int8_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT8: { + using T = uint8_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_INT16: { + using T = int16_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT16: { + using T = uint16_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT32: { + using T = uint32_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_UINT64: { + using T = uint64_t; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_FLOAT32: { + using T = float; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_FLOAT64: { + using T = double; + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0.cast(), r1.cast()); + break; + } + case TILEDB_STRING_ASCII: + case TILEDB_STRING_UTF8: + case TILEDB_CHAR: { + if (!py::isinstance(r0) != !py::isinstance(r1)) { + TPY_ERROR_LOC( + "internal error: ranges must both be strings or (None, None)"); + } else if ( + !py::isinstance(r0) && !py::isinstance(r1) && + !py::isinstance(r0) && !py::isinstance(r1) && + !py::isinstance(r0) && !py::isinstance(r1)) { + TPY_ERROR_LOC( + "internal error: expected string type for var-length label!"); + } + + if (!py::isinstance(r0) && !py::isinstance(r0)) { + std::string r0_string = r0.cast(); + std::string r1_string = r1.cast(); + SubarrayExperimental::add_label_range( + ctx, subarray, label_name, r0_string, r1_string); + } + break; + } + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_SEC: + case TILEDB_DATETIME_MS: + case TILEDB_DATETIME_US: + case TILEDB_DATETIME_NS: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: { + case TILEDB_TIME_HR: + case TILEDB_TIME_MIN: + case TILEDB_TIME_SEC: + case TILEDB_TIME_MS: + case TILEDB_TIME_US: + case TILEDB_TIME_NS: + case TILEDB_TIME_PS: + case TILEDB_TIME_FS: + case TILEDB_TIME_AS: + py::dtype dtype = tdb_to_np_dtype(tiledb_type, 1); + auto dt0 = + py::isinstance(r0) ? r0 : r0.attr("astype")(dtype); + auto dt1 = + py::isinstance(r1) ? r1 : r1.attr("astype")(dtype); + + if (py::isinstance(dt0) && py::isinstance(dt1)) { + SubarrayExperimental::add_label_range( + ctx, + subarray, + label_name, + py::cast(dt0), + py::cast(dt1)); + } else { + auto darray = py::array(py::make_tuple(dt0, dt1)); + SubarrayExperimental::add_label_range( + ctx, + subarray, + label_name, + *(int64_t*)darray.data(0), + *(int64_t*)darray.data(1)); + } + + break; + } + default: + TPY_ERROR_LOC("Unknown dimension label type conversion!"); } - } catch (py::cast_error &e) { + } catch (py::cast_error& e) { (void)e; std::string msg = "Failed to cast label range '" + (std::string)py::repr(r) + "' to label type " + @@ -528,7 +541,7 @@ void add_label_range(const Context &ctx, Subarray &subarray, } } -bool has_label_range(const Context &ctx, Subarray &subarray, uint32_t dim_idx) { +bool has_label_range(const Context& ctx, Subarray& subarray, uint32_t dim_idx) { int32_t has_label; auto rc = tiledb_subarray_has_label_ranges( ctx.ptr().get(), subarray.ptr().get(), dim_idx, &has_label); @@ -538,121 +551,141 @@ bool has_label_range(const Context &ctx, Subarray &subarray, uint32_t dim_idx) { return has_label == 1; } -void init_subarray(py::module &m) { +void init_subarray(py::module& m) { py::class_(m, "Subarray") .def(py::init()) - .def(py::init(), - py::keep_alive<1, 2>() /* Keep context alive. */, - py::keep_alive<1, 3>() /* Keep array alive. */) - - .def("__capsule__", - [](Subarray &subarray) { - return py::capsule(subarray.ptr().get(), "subarray"); - }) - - .def("_add_dim_range", - [](Subarray &subarray, uint32_t dim_idx, py::tuple range) { - add_dim_range(subarray, dim_idx, range); - }) - - .def("_add_label_range", - [](Subarray &subarray, const Context &ctx, - const std::string &label_name, py::tuple range) { - add_label_range(ctx, subarray, label_name, range); - }) - - .def("_add_ranges_bulk", - [](Subarray &subarray, const Context &ctx, py::iterable ranges) { - uint32_t dim_idx = 0; - for (auto dim_range : ranges) { - if (py::isinstance(dim_range)) { - add_dim_point_ranges(ctx, subarray, dim_idx, dim_range); - } else { - py::tuple dim_range_iter = dim_range.cast(); - for (auto r : dim_range_iter) { - py::tuple range_tuple = r.cast(); - add_dim_range(subarray, dim_idx, range_tuple); - } - } - dim_idx++; - } - }) - - .def("_add_dim_point_ranges", - [](Subarray &subarray, const Context &ctx, uint32_t dim_idx, - pybind11::handle dim_range) { - add_dim_point_ranges(ctx, subarray, dim_idx, dim_range); - }) - - .def("_add_ranges", - [](Subarray &subarray, const Context &ctx, py::iterable ranges) { - uint32_t dim_idx = 0; - for (auto dim_range : ranges) { - py::tuple dim_range_iter = dim_range.cast(); - for (auto r : dim_range_iter) { - py::tuple r_tuple = r.cast(); - add_dim_range(subarray, dim_idx, r_tuple); - } - dim_idx++; - } - }) - - .def("_add_label_ranges", - [](Subarray &subarray, const Context &ctx, py::iterable ranges) { - py::dict label_ranges = ranges.cast(); - for (std::pair pair : label_ranges) { - py::str label_name = pair.first.cast(); - py::tuple label_range_iter = pair.second.cast(); - for (auto r : label_range_iter) { - py::tuple r_tuple = r.cast(); - add_label_range(ctx, subarray, label_name, r_tuple); - } - } - }) - - .def("_has_label_range", - [](Subarray &subarray, const Context &ctx, uint32_t dim_idx) { - return has_label_range(ctx, subarray, dim_idx); - }) - - .def("copy_ranges", - [](Subarray &subarray, Subarray &original, py::iterable dims) { - for (auto dim_idx : dims) { - copy_ranges_on_dim(subarray, original, dim_idx.cast()); - } - }) - - .def("_range_num", py::overload_cast( - &Subarray::range_num, py::const_)) - - .def("_range_num", - py::overload_cast(&Subarray::range_num, py::const_)) - - .def("_label_range_num", - [](Subarray &subarray, const Context &ctx, - const std::string &label_name) { - return SubarrayExperimental::label_range_num(ctx, subarray, - label_name); - }) - - .def("_shape", - [](Subarray &subarray, const Context &ctx) { - auto ndim = subarray.array().schema().domain().ndim(); - // Create numpy array and get pointer to data. - py::array_t shape(ndim); - py::buffer_info shape_result = shape.request(); - py::ssize_t *shape_ptr = - static_cast(shape_result.ptr); - // Set size for each dimension. - for (uint32_t dim_idx{0}; dim_idx < ndim; ++dim_idx) { - shape_ptr[dim_idx] = length_ranges(subarray, dim_idx); - } - return shape; - }) + .def( + py::init(), + py::keep_alive<1, 2>() /* Keep context alive. */, + py::keep_alive<1, 3>() /* Keep array alive. */) + + .def( + "__capsule__", + [](Subarray& subarray) { + return py::capsule(subarray.ptr().get(), "subarray"); + }) + + .def( + "_add_dim_range", + [](Subarray& subarray, uint32_t dim_idx, py::tuple range) { + add_dim_range(subarray, dim_idx, range); + }) + + .def( + "_add_label_range", + [](Subarray& subarray, + const Context& ctx, + const std::string& label_name, + py::tuple range) { + add_label_range(ctx, subarray, label_name, range); + }) + + .def( + "_add_ranges_bulk", + [](Subarray& subarray, const Context& ctx, py::iterable ranges) { + uint32_t dim_idx = 0; + for (auto dim_range : ranges) { + if (py::isinstance(dim_range)) { + add_dim_point_ranges(ctx, subarray, dim_idx, dim_range); + } else { + py::tuple dim_range_iter = dim_range.cast(); + for (auto r : dim_range_iter) { + py::tuple range_tuple = r.cast(); + add_dim_range(subarray, dim_idx, range_tuple); + } + } + dim_idx++; + } + }) + + .def( + "_add_dim_point_ranges", + [](Subarray& subarray, + const Context& ctx, + uint32_t dim_idx, + pybind11::handle dim_range) { + add_dim_point_ranges(ctx, subarray, dim_idx, dim_range); + }) + + .def( + "_add_ranges", + [](Subarray& subarray, const Context& ctx, py::iterable ranges) { + uint32_t dim_idx = 0; + for (auto dim_range : ranges) { + py::tuple dim_range_iter = dim_range.cast(); + for (auto r : dim_range_iter) { + py::tuple r_tuple = r.cast(); + add_dim_range(subarray, dim_idx, r_tuple); + } + dim_idx++; + } + }) + + .def( + "_add_label_ranges", + [](Subarray& subarray, const Context& ctx, py::iterable ranges) { + py::dict label_ranges = ranges.cast(); + for (std::pair pair : label_ranges) { + py::str label_name = pair.first.cast(); + py::tuple label_range_iter = pair.second.cast(); + for (auto r : label_range_iter) { + py::tuple r_tuple = r.cast(); + add_label_range(ctx, subarray, label_name, r_tuple); + } + } + }) + + .def( + "_has_label_range", + [](Subarray& subarray, const Context& ctx, uint32_t dim_idx) { + return has_label_range(ctx, subarray, dim_idx); + }) + + .def( + "copy_ranges", + [](Subarray& subarray, Subarray& original, py::iterable dims) { + for (auto dim_idx : dims) { + copy_ranges_on_dim(subarray, original, dim_idx.cast()); + } + }) + + .def( + "_range_num", + py::overload_cast( + &Subarray::range_num, py::const_)) + + .def( + "_range_num", + py::overload_cast(&Subarray::range_num, py::const_)) + + .def( + "_label_range_num", + [](Subarray& subarray, + const Context& ctx, + const std::string& label_name) { + return SubarrayExperimental::label_range_num( + ctx, subarray, label_name); + }) + + .def( + "_shape", + [](Subarray& subarray, const Context& ctx) { + auto ndim = subarray.array().schema().domain().ndim(); + // Create numpy array and get pointer to data. + py::array_t shape(ndim); + py::buffer_info shape_result = shape.request(); + py::ssize_t* shape_ptr = + static_cast(shape_result.ptr); + // Set size for each dimension. + for (uint32_t dim_idx{0}; dim_idx < ndim; ++dim_idx) { + shape_ptr[dim_idx] = length_ranges(subarray, dim_idx); + } + return shape; + }) // End definitions. ; } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/cc/tiledbcpp.cc b/tiledb/cc/tiledbcpp.cc index cce6e433f6..83cc08a01c 100644 --- a/tiledb/cc/tiledbcpp.cc +++ b/tiledb/cc/tiledbcpp.cc @@ -1,10 +1,10 @@ -#include // C++ +#include // C++ -#include "common.h" #include #include #include #include +#include "common.h" namespace libtiledbcpp { @@ -14,28 +14,27 @@ namespace py = pybind11; template using overload_cast_ = pybind11::detail::overload_cast_impl; -void init_array(py::module &); -void init_attribute(py::module &); -void init_context(py::module &); -void init_config(py::module &); -void init_consolidation_plan(py::module &m); -void init_current_domain(py::module &m); -void init_enums(py::module &); -void init_enumeration(py::module &); -void init_dimension_label(py::module &m); -void init_domain(py::module &m); -void init_file_handle(py::module &); -void init_filestore(py::module &m); -void init_filter(py::module &); -void init_group(py::module &); -void init_object(py::module &m); -void init_query(py::module &m); -void init_schema(py::module &); -void init_subarray(py::module &); -void init_vfs(py::module &m); +void init_array(py::module&); +void init_attribute(py::module&); +void init_context(py::module&); +void init_config(py::module&); +void init_consolidation_plan(py::module& m); +void init_current_domain(py::module& m); +void init_enums(py::module&); +void init_enumeration(py::module&); +void init_dimension_label(py::module& m); +void init_domain(py::module& m); +void init_file_handle(py::module&); +void init_filestore(py::module& m); +void init_filter(py::module&); +void init_group(py::module&); +void init_object(py::module& m); +void init_query(py::module& m); +void init_schema(py::module&); +void init_subarray(py::module&); +void init_vfs(py::module& m); PYBIND11_MODULE(cc, m) { - init_array(m); init_attribute(m); init_context(m); @@ -70,14 +69,14 @@ PYBIND11_MODULE(cc, m) { try { if (p) std::rethrow_exception(p); - } catch (const TileDBPyError &e) { + } catch (const TileDBPyError& e) { PyErr_SetString(tiledb_py_error.ptr(), e.what()); - } catch (const tiledb::TileDBError &e) { + } catch (const tiledb::TileDBError& e) { PyErr_SetString(tiledb_py_error.ptr(), e.what()); - } catch (py::builtin_exception &e) { + } catch (py::builtin_exception& e) { throw; }; }); } -}; // namespace libtiledbcpp +}; // namespace libtiledbcpp diff --git a/tiledb/cc/vfs.cc b/tiledb/cc/vfs.cc index debf87df1d..88d673f362 100644 --- a/tiledb/cc/vfs.cc +++ b/tiledb/cc/vfs.cc @@ -15,10 +15,10 @@ using namespace tiledb; using namespace tiledbpy::common; namespace py = pybind11; -void init_vfs(py::module &m) { +void init_vfs(py::module& m) { py::class_(m, "VFS") - .def(py::init(), py::keep_alive<1, 2>()) - .def(py::init(), py::keep_alive<1, 2>()) + .def(py::init(), py::keep_alive<1, 2>()) + .def(py::init(), py::keep_alive<1, 2>()) .def_property_readonly("_ctx", &VFS::context) .def_property_readonly("_config", &VFS::config) @@ -49,12 +49,14 @@ void init_vfs(py::module &m) { // C++ function. Return an empty vector. // 2. no callback is passed and a default callback is used. The // default callback just appends each path gathered. Return the paths. - [](VFS &vfs, std::string uri, py::object callback) { + [](VFS& vfs, std::string uri, py::object callback) { tiledb::Context ctx; if (callback.is_none()) { std::vector paths; tiledb::VFSExperimental::ls_recursive( - ctx, vfs, uri, + ctx, + vfs, + uri, [&](const std::string_view path, uint64_t object_size) -> bool { paths.push_back(std::string(path)); @@ -63,7 +65,9 @@ void init_vfs(py::module &m) { return paths; } else { tiledb::VFSExperimental::ls_recursive( - ctx, vfs, uri, + ctx, + vfs, + uri, [&](const std::string_view path, uint64_t object_size) -> bool { return callback(path, object_size).cast(); @@ -75,16 +79,19 @@ void init_vfs(py::module &m) { } class FileHandle { -private: + private: Context _ctx; - tiledb_vfs_fh_t *_fh; - -public: - FileHandle(const Context &ctx, const VFS &vfs, std::string uri, - tiledb_vfs_mode_t mode) + tiledb_vfs_fh_t* _fh; + + public: + FileHandle( + const Context& ctx, + const VFS& vfs, + std::string uri, + tiledb_vfs_mode_t mode) : _ctx(ctx) { - _ctx.handle_error(tiledb_vfs_open(_ctx.ptr().get(), vfs.ptr().get(), - uri.c_str(), mode, &this->_fh)); + _ctx.handle_error(tiledb_vfs_open( + _ctx.ptr().get(), vfs.ptr().get(), uri.c_str(), mode, &this->_fh)); } void close() { @@ -95,8 +102,8 @@ class FileHandle { py::array data = py::array(py::dtype::of(), nbytes); py::buffer_info buffer = data.request(); - _ctx.handle_error(tiledb_vfs_read(_ctx.ptr().get(), this->_fh, offset, - buffer.ptr, nbytes)); + _ctx.handle_error(tiledb_vfs_read( + _ctx.ptr().get(), this->_fh, offset, buffer.ptr, nbytes)); auto np = py::module::import("numpy"); auto to_bytes = np.attr("ndarray").attr("tobytes"); @@ -106,8 +113,8 @@ class FileHandle { void write(py::buffer data) { py::buffer_info buffer = data.request(); - _ctx.handle_error(tiledb_vfs_write(_ctx.ptr().get(), this->_fh, buffer.ptr, - buffer.shape[0])); + _ctx.handle_error(tiledb_vfs_write( + _ctx.ptr().get(), this->_fh, buffer.ptr, buffer.shape[0])); } void flush() { @@ -122,11 +129,15 @@ class FileHandle { } }; -void init_file_handle(py::module &m) { +void init_file_handle(py::module& m) { py::class_(m, "FileHandle") - .def(py::init(), - py::keep_alive<1, 2>()) + .def( + py::init< + const Context&, + const VFS&, + std::string, + tiledb_vfs_mode_t>(), + py::keep_alive<1, 2>()) .def_property_readonly("_closed", &FileHandle::closed) @@ -136,4 +147,4 @@ void init_file_handle(py::module &m) { .def("_flush", &FileHandle::flush); } -} // namespace libtiledbcpp +} // namespace libtiledbcpp diff --git a/tiledb/core.cc b/tiledb/core.cc index 8a5f778f7a..2cb4c720c3 100644 --- a/tiledb/core.cc +++ b/tiledb/core.cc @@ -9,17 +9,17 @@ #include #include -#include "npbuffer.h" -#include "util.h" #include #include #include #include +#include "npbuffer.h" +#include "util.h" -#include // C -#include // C++ -#include // C -#include // C++ +#include // C +#include // C +#include // C++ +#include // C++ // clang-format off // do not re-order these headers @@ -28,7 +28,7 @@ // clang-format on #if defined(TILEDB_SERIALIZATION) -#include // C +#include // C #endif #include "../external/tsl/robin_map.h" @@ -80,19 +80,28 @@ static std::unique_ptr g_stats; py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num); struct BufferInfo { - BufferInfo(std::string name, size_t data_nbytes, tiledb_datatype_t data_type, - uint32_t cell_val_num, size_t offsets_num, size_t validity_num, - bool isvar = false, bool isnullable = false) - - : name(name), type(data_type), cell_val_num(cell_val_num), isvar(isvar), - isnullable(isnullable) { + BufferInfo( + std::string name, + size_t data_nbytes, + tiledb_datatype_t data_type, + uint32_t cell_val_num, + size_t offsets_num, + size_t validity_num, + bool isvar = false, + bool isnullable = false) + + : name(name) + , type(data_type) + , cell_val_num(cell_val_num) + , isvar(isvar) + , isnullable(isnullable) { try { dtype = tiledb_dtype(data_type, cell_val_num); elem_nbytes = tiledb_datatype_size(type); data = py::array(py::dtype("uint8"), data_nbytes); offsets = py::array_t(offsets_num); validity = py::array_t(validity_num); - } catch (py::error_already_set &e) { + } catch (py::error_already_set& e) { TPY_ERROR_LOC(e.what()) } // TODO use memset here for zero'd buffers in debug mode @@ -120,9 +129,14 @@ struct BufferHolder { py::object validity; BufferHolder(py::object d, py::object o, py::object v) - : data(d), offsets(o), validity(v) {} + : data(d) + , offsets(o) + , validity(v) { + } - static void free_buffer_holder(BufferHolder *self) { delete self; } + static void free_buffer_holder(BufferHolder* self) { + delete self; + } }; py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) { @@ -130,115 +144,116 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) { auto np = py::module::import("numpy"); auto datetime64 = np.attr("datetime64"); switch (type) { - case TILEDB_INT32: - return py::dtype("int32"); - case TILEDB_INT64: - return py::dtype("int64"); - case TILEDB_FLOAT32: - return py::dtype("float32"); - case TILEDB_FLOAT64: - return py::dtype("float64"); - case TILEDB_INT8: - return py::dtype("int8"); - case TILEDB_UINT8: - return py::dtype("uint8"); - case TILEDB_INT16: - return py::dtype("int16"); - case TILEDB_UINT16: - return py::dtype("uint16"); - case TILEDB_UINT32: - return py::dtype("uint32"); - case TILEDB_UINT64: - return py::dtype("uint64"); - case TILEDB_STRING_ASCII: - return py::dtype("S1"); - case TILEDB_STRING_UTF8: - return py::dtype("U1"); - case TILEDB_STRING_UTF16: - case TILEDB_STRING_UTF32: - TPY_ERROR_LOC("Unimplemented UTF16 or UTF32 string conversion!"); - case TILEDB_STRING_UCS2: - case TILEDB_STRING_UCS4: - TPY_ERROR_LOC("Unimplemented UCS2 or UCS4 string conversion!"); - case TILEDB_CHAR: - return py::dtype("S1"); - case TILEDB_DATETIME_YEAR: - return py::dtype("M8[Y]"); - case TILEDB_DATETIME_MONTH: - return py::dtype("M8[M]"); - case TILEDB_DATETIME_WEEK: - return py::dtype("M8[W]"); - case TILEDB_DATETIME_DAY: - return py::dtype("M8[D]"); - case TILEDB_DATETIME_HR: - return py::dtype("M8[h]"); - case TILEDB_DATETIME_MIN: - return py::dtype("M8[m]"); - case TILEDB_DATETIME_SEC: - return py::dtype("M8[s]"); - case TILEDB_DATETIME_MS: - return py::dtype("M8[ms]"); - case TILEDB_DATETIME_US: - return py::dtype("M8[us]"); - case TILEDB_DATETIME_NS: - return py::dtype("M8[ns]"); - case TILEDB_DATETIME_PS: - return py::dtype("M8[ps]"); - case TILEDB_DATETIME_FS: - return py::dtype("M8[fs]"); - case TILEDB_DATETIME_AS: - return py::dtype("M8[as]"); - - /* duration types map to timedelta */ - case TILEDB_TIME_HR: - return py::dtype("m8[h]"); - case TILEDB_TIME_MIN: - return py::dtype("m8[m]"); - case TILEDB_TIME_SEC: - return py::dtype("m8[s]"); - case TILEDB_TIME_MS: - return py::dtype("m8[ms]"); - case TILEDB_TIME_US: - return py::dtype("m8[us]"); - case TILEDB_TIME_NS: - return py::dtype("m8[ns]"); - case TILEDB_TIME_PS: - return py::dtype("m8[ps]"); - case TILEDB_TIME_FS: - return py::dtype("m8[fs]"); - case TILEDB_TIME_AS: - return py::dtype("m8[as]"); - case TILEDB_BLOB: - return py::dtype("byte"); - case TILEDB_BOOL: - return py::dtype("bool"); + case TILEDB_INT32: + return py::dtype("int32"); + case TILEDB_INT64: + return py::dtype("int64"); + case TILEDB_FLOAT32: + return py::dtype("float32"); + case TILEDB_FLOAT64: + return py::dtype("float64"); + case TILEDB_INT8: + return py::dtype("int8"); + case TILEDB_UINT8: + return py::dtype("uint8"); + case TILEDB_INT16: + return py::dtype("int16"); + case TILEDB_UINT16: + return py::dtype("uint16"); + case TILEDB_UINT32: + return py::dtype("uint32"); + case TILEDB_UINT64: + return py::dtype("uint64"); + case TILEDB_STRING_ASCII: + return py::dtype("S1"); + case TILEDB_STRING_UTF8: + return py::dtype("U1"); + case TILEDB_STRING_UTF16: + case TILEDB_STRING_UTF32: + TPY_ERROR_LOC("Unimplemented UTF16 or UTF32 string conversion!"); + case TILEDB_STRING_UCS2: + case TILEDB_STRING_UCS4: + TPY_ERROR_LOC("Unimplemented UCS2 or UCS4 string conversion!"); + case TILEDB_CHAR: + return py::dtype("S1"); + case TILEDB_DATETIME_YEAR: + return py::dtype("M8[Y]"); + case TILEDB_DATETIME_MONTH: + return py::dtype("M8[M]"); + case TILEDB_DATETIME_WEEK: + return py::dtype("M8[W]"); + case TILEDB_DATETIME_DAY: + return py::dtype("M8[D]"); + case TILEDB_DATETIME_HR: + return py::dtype("M8[h]"); + case TILEDB_DATETIME_MIN: + return py::dtype("M8[m]"); + case TILEDB_DATETIME_SEC: + return py::dtype("M8[s]"); + case TILEDB_DATETIME_MS: + return py::dtype("M8[ms]"); + case TILEDB_DATETIME_US: + return py::dtype("M8[us]"); + case TILEDB_DATETIME_NS: + return py::dtype("M8[ns]"); + case TILEDB_DATETIME_PS: + return py::dtype("M8[ps]"); + case TILEDB_DATETIME_FS: + return py::dtype("M8[fs]"); + case TILEDB_DATETIME_AS: + return py::dtype("M8[as]"); + + /* duration types map to timedelta */ + case TILEDB_TIME_HR: + return py::dtype("m8[h]"); + case TILEDB_TIME_MIN: + return py::dtype("m8[m]"); + case TILEDB_TIME_SEC: + return py::dtype("m8[s]"); + case TILEDB_TIME_MS: + return py::dtype("m8[ms]"); + case TILEDB_TIME_US: + return py::dtype("m8[us]"); + case TILEDB_TIME_NS: + return py::dtype("m8[ns]"); + case TILEDB_TIME_PS: + return py::dtype("m8[ps]"); + case TILEDB_TIME_FS: + return py::dtype("m8[fs]"); + case TILEDB_TIME_AS: + return py::dtype("m8[as]"); + case TILEDB_BLOB: + return py::dtype("byte"); + case TILEDB_BOOL: + return py::dtype("bool"); #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 21 - case TILEDB_GEOM_WKB: - return py::dtype("byte"); - case TILEDB_GEOM_WKT: - return py::dtype("S"); + case TILEDB_GEOM_WKB: + return py::dtype("byte"); + case TILEDB_GEOM_WKT: + return py::dtype("S"); #endif - case TILEDB_ANY: - break; + case TILEDB_ANY: + break; } } else if (cell_val_num == 2 && type == TILEDB_FLOAT32) { return py::dtype("complex64"); } else if (cell_val_num == 2 && type == TILEDB_FLOAT64) { return py::dtype("complex128"); - } else if (type == TILEDB_CHAR || type == TILEDB_STRING_UTF8 || - type == TILEDB_STRING_ASCII) { + } else if ( + type == TILEDB_CHAR || type == TILEDB_STRING_UTF8 || + type == TILEDB_STRING_ASCII) { std::string base_str; switch (type) { - case TILEDB_CHAR: - case TILEDB_STRING_ASCII: - base_str = "|S"; - break; - case TILEDB_STRING_UTF8: - base_str = "|U"; - break; - default: - TPY_ERROR_LOC("internal error: unhandled string type"); + case TILEDB_CHAR: + case TILEDB_STRING_ASCII: + base_str = "|S"; + break; + case TILEDB_STRING_UTF8: + base_str = "|U"; + break; + default: + TPY_ERROR_LOC("internal error: unhandled string type"); } if (cell_val_num < TILEDB_VAR_NUM) { base_str = base_str + std::to_string(cell_val_num); @@ -258,13 +273,13 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) { return np_dtype(rec_list); } - TPY_ERROR_LOC("tiledb datatype not understood ('" + - tiledb::impl::type_to_str(type) + - "', cell_val_num: " + std::to_string(cell_val_num) + ")"); + TPY_ERROR_LOC( + "tiledb datatype not understood ('" + tiledb::impl::type_to_str(type) + + "', cell_val_num: " + std::to_string(cell_val_num) + ")"); } -py::array_t -uint8_bool_to_uint8_bitmap(py::array_t validity_array) { +py::array_t uint8_bool_to_uint8_bitmap( + py::array_t validity_array) { // TODO profile, probably replace; avoid inplace reassignment auto np = py::module::import("numpy"); auto packbits = np.attr("packbits"); @@ -284,7 +299,7 @@ class PyAgg { using AggToBufferMap = std::map; using AttrToAggsMap = std::map; -private: + private: Context ctx_; std::shared_ptr array_; std::shared_ptr query_; @@ -294,13 +309,17 @@ class PyAgg { py::dict original_input_; std::vector attrs_; -public: + public: PyAgg() = delete; - PyAgg(const Context &ctx, py::object py_array, py::object py_layout, - py::dict attr_to_aggs_input) - : ctx_(ctx), original_input_(attr_to_aggs_input) { - tiledb_array_t *c_array_ = (py::capsule)py_array.attr("__capsule__")(); + PyAgg( + const Context& ctx, + py::object py_array, + py::object py_layout, + py::dict attr_to_aggs_input) + : ctx_(ctx) + , original_input_(attr_to_aggs_input) { + tiledb_array_t* c_array_ = (py::capsule)py_array.attr("__capsule__")(); // We never own this pointer; pass own=false array_ = std::make_shared(ctx_, c_array_, false); @@ -341,7 +360,7 @@ class PyAgg { _apply_agg_operator_to_attr(agg_name, attr_name); // Set the result data buffers - auto *res_buf = &result_buffers_[attr_name][agg_name]; + auto* res_buf = &result_buffers_[attr_name][agg_name]; if ("count" == agg_name || "null_count" == agg_name || "mean" == agg_name) { // count and null_count use uint64 and mean uses float64 @@ -351,8 +370,8 @@ class PyAgg { py::dtype dt(tiledb_dtype(attr.type(), attr.cell_size())); *res_buf = py::array(py::dtype("uint8"), dt.itemsize()); } - query_->set_data_buffer(attr_name + agg_name, (void *)res_buf->data(), - 1); + query_->set_data_buffer( + attr_name + agg_name, (void*)res_buf->data(), 1); if (attr.nullable()) { // For nullable attributes, if the input set for the aggregation @@ -360,20 +379,20 @@ class PyAgg { // as this operation is undefined. We need to check the validity // buffer beforehand to see if we had a valid result if (!("count" == agg_name || "null_count" == agg_name)) { - auto *val_buf = &validity_buffers_[attr.name()][agg_name]; + auto* val_buf = &validity_buffers_[attr.name()][agg_name]; *val_buf = py::array(py::dtype("uint8"), 1); - query_->set_validity_buffer(attr_name + agg_name, - (uint8_t *)val_buf->data(), 1); + query_->set_validity_buffer( + attr_name + agg_name, (uint8_t*)val_buf->data(), 1); } } } } } - void _apply_agg_operator_to_attr(const std::string &op_label, - const std::string &attr_name) { + void _apply_agg_operator_to_attr( + const std::string& op_label, const std::string& attr_name) { using AggregateFunc = - std::function; + std::function; std::unordered_map label_to_agg_func = { {"sum", QueryExperimental::create_unary_aggregate}, @@ -394,8 +413,9 @@ class PyAgg { } else if ("count" == op_label) { default_channel.apply_aggregate(attr_name + op_label, CountOperation()); } else { - TPY_ERROR_LOC("Invalid channel operation " + op_label + - " passed to apply_aggregate."); + TPY_ERROR_LOC( + "Invalid channel operation " + op_label + + " passed to apply_aggregate."); } } @@ -434,14 +454,14 @@ class PyAgg { return false; // For nullable attributes, check if the validity buffer returned false - const void *val_buf = validity_buffers_[attr.name()][agg_name].data(); - return *((uint8_t *)(val_buf)) == 0; + const void* val_buf = validity_buffers_[attr.name()][agg_name].data(); + return *((uint8_t*)(val_buf)) == 0; } else { // For non-nullable attributes, max and min are undefined for the empty // set, so we must check the count == 0 if ("max" == agg_name || "min" == agg_name) { - const void *count_buf = result_buffers_[attr.name()]["count"].data(); - return *((uint64_t *)(count_buf)) == 0; + const void* count_buf = result_buffers_[attr.name()]["count"].data(); + return *((uint64_t*)(count_buf)) == 0; } return false; } @@ -449,59 +469,59 @@ class PyAgg { bool _is_integer_dtype(tiledb::Attribute attr) { switch (attr.type()) { - case TILEDB_INT8: - case TILEDB_INT16: - case TILEDB_UINT8: - case TILEDB_INT32: - case TILEDB_INT64: - case TILEDB_UINT16: - case TILEDB_UINT32: - case TILEDB_UINT64: - return true; - default: - return false; + case TILEDB_INT8: + case TILEDB_INT16: + case TILEDB_UINT8: + case TILEDB_INT32: + case TILEDB_INT64: + case TILEDB_UINT16: + case TILEDB_UINT32: + case TILEDB_UINT64: + return true; + default: + return false; } } py::object _set_result(tiledb::Attribute attr, std::string agg_name) { - const void *agg_buf = result_buffers_[attr.name()][agg_name].data(); + const void* agg_buf = result_buffers_[attr.name()][agg_name].data(); if ("mean" == agg_name) - return py::cast(*((double *)agg_buf)); + return py::cast(*((double*)agg_buf)); if ("count" == agg_name || "null_count" == agg_name) - return py::cast(*((uint64_t *)agg_buf)); + return py::cast(*((uint64_t*)agg_buf)); switch (attr.type()) { - case TILEDB_FLOAT32: - return py::cast("sum" == agg_name ? *((double *)agg_buf) - : *((float *)agg_buf)); - case TILEDB_FLOAT64: - return py::cast(*((double *)agg_buf)); - case TILEDB_INT8: - return py::cast(*((int8_t *)agg_buf)); - case TILEDB_UINT8: - return py::cast(*((uint8_t *)agg_buf)); - case TILEDB_INT16: - return py::cast(*((int16_t *)agg_buf)); - case TILEDB_UINT16: - return py::cast(*((uint16_t *)agg_buf)); - case TILEDB_UINT32: - return py::cast(*((uint32_t *)agg_buf)); - case TILEDB_INT32: - return py::cast(*((int32_t *)agg_buf)); - case TILEDB_INT64: - return py::cast(*((int64_t *)agg_buf)); - case TILEDB_UINT64: - return py::cast(*((uint64_t *)agg_buf)); - default: - TPY_ERROR_LOC( - "[_cast_agg_result] Invalid tiledb dtype for aggregation result") + case TILEDB_FLOAT32: + return py::cast( + "sum" == agg_name ? *((double*)agg_buf) : *((float*)agg_buf)); + case TILEDB_FLOAT64: + return py::cast(*((double*)agg_buf)); + case TILEDB_INT8: + return py::cast(*((int8_t*)agg_buf)); + case TILEDB_UINT8: + return py::cast(*((uint8_t*)agg_buf)); + case TILEDB_INT16: + return py::cast(*((int16_t*)agg_buf)); + case TILEDB_UINT16: + return py::cast(*((uint16_t*)agg_buf)); + case TILEDB_UINT32: + return py::cast(*((uint32_t*)agg_buf)); + case TILEDB_INT32: + return py::cast(*((int32_t*)agg_buf)); + case TILEDB_INT64: + return py::cast(*((int64_t*)agg_buf)); + case TILEDB_UINT64: + return py::cast(*((uint64_t*)agg_buf)); + default: + TPY_ERROR_LOC( + "[_cast_agg_result] Invalid tiledb dtype for aggregation result") } } void set_subarray(py::object py_subarray) { - query_->set_subarray(*py_subarray.cast()); + query_->set_subarray(*py_subarray.cast()); } void set_cond(py::object cond) { @@ -509,9 +529,9 @@ class PyAgg { try { init_pyqc(array_->uri(), attrs_, ctx_); - } catch (tiledb::TileDBError &e) { + } catch (tiledb::TileDBError& e) { TPY_ERROR_LOC(e.what()); - } catch (py::error_already_set &e) { + } catch (py::error_already_set& e) { TPY_ERROR_LOC(e.what()); } auto pyqc = (cond.attr("c_obj")).cast(); @@ -521,7 +541,7 @@ class PyAgg { }; class PyQuery { -private: + private: Context ctx_; std::shared_ptr domain_; std::shared_ptr array_schema_; @@ -544,18 +564,23 @@ class PyQuery { // label buffer list unordered_map label_input_buffer_data_; -public: - tiledb_ctx_t *c_ctx_; - tiledb_array_t *c_array_; + public: + tiledb_ctx_t* c_ctx_; + tiledb_array_t* c_array_; bool preload_metadata_ = false; bool return_incomplete_ = false; size_t retries_ = 0; -public: + public: PyQuery() = delete; - PyQuery(const Context &ctx, py::object array, py::iterable attrs, - py::iterable dims, py::object py_layout, py::object use_arrow) + PyQuery( + const Context& ctx, + py::object array, + py::iterable attrs, + py::iterable dims, + py::object py_layout, + py::object use_arrow) : ctx_(ctx) { init_config(); // initialize arrow argument from user, if provided @@ -564,7 +589,7 @@ class PyQuery { use_arrow_ = py::cast(use_arrow); } - tiledb_array_t *c_array_ = (py::capsule)array.attr("__capsule__")(); + tiledb_array_t* c_array_ = (py::capsule)array.attr("__capsule__")(); // we never own this pointer, pass own=false array_ = std::make_shared(ctx_, c_array_, false); @@ -637,23 +662,23 @@ class PyQuery { } void set_subarray(py::object py_subarray) { - query_->set_subarray(*py_subarray.cast()); + query_->set_subarray(*py_subarray.cast()); } #if defined(TILEDB_SERIALIZATION) void set_serialized_query(py::buffer serialized_query) { int rc; - tiledb_query_t *c_query; - tiledb_buffer_t *c_buffer; - tiledb_ctx_t *c_ctx = ctx_.ptr().get(); + tiledb_query_t* c_query; + tiledb_buffer_t* c_buffer; + tiledb_ctx_t* c_ctx = ctx_.ptr().get(); rc = tiledb_buffer_alloc(c_ctx, &c_buffer); if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not allocate c_buffer."); py::buffer_info buffer_info = serialized_query.request(); - rc = tiledb_buffer_set_data(c_ctx, c_buffer, buffer_info.ptr, - buffer_info.shape[0]); + rc = tiledb_buffer_set_data( + c_ctx, c_buffer, buffer_info.ptr, buffer_info.shape[0]); if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not set c_buffer."); @@ -669,9 +694,9 @@ class PyQuery { try { init_pyqc(array_->uri(), attrs_, ctx_); - } catch (tiledb::TileDBError &e) { + } catch (tiledb::TileDBError& e) { TPY_ERROR_LOC(e.what()); - } catch (py::error_already_set &e) { + } catch (py::error_already_set& e) { TPY_ERROR_LOC(e.what()); } auto pyqc = (cond.attr("c_obj")).cast(); @@ -679,15 +704,17 @@ class PyQuery { query_->set_condition(*qc); } - bool is_dimension(std::string name) { return domain_->has_dimension(name); } + bool is_dimension(std::string name) { + return domain_->has_dimension(name); + } bool is_attribute(std::string name) { return array_schema_->has_attribute(name); } bool is_dimension_label(std::string name) { - return ArraySchemaExperimental::has_dimension_label(ctx_, *array_schema_, - name); + return ArraySchemaExperimental::has_dimension_label( + ctx_, *array_schema_, name); } bool is_var(std::string name) { @@ -702,8 +729,9 @@ class PyQuery { ArraySchemaExperimental::dimension_label(ctx_, *array_schema_, name); return dim_label.label_cell_val_num() == TILEDB_VAR_NUM; } else { - TPY_ERROR_LOC("Unknown buffer type for is_var check (expected attribute " - "or dimension)") + TPY_ERROR_LOC( + "Unknown buffer type for is_var check (expected attribute " + "or dimension)") } } @@ -749,13 +777,15 @@ class PyQuery { try { auto t = buffer_type(name); return tiledb_dtype(t.first, t.second); - } catch (TileDBError &e) { + } catch (TileDBError& e) { (void)e; return py::none(); } } - bool is_sparse() { return array_->schema().array_type() == TILEDB_SPARSE; } + bool is_sparse() { + return array_->schema().array_type() == TILEDB_SPARSE; + } void import_buffer(std::string name, py::array data, py::array offsets) { tiledb_datatype_t type; @@ -765,9 +795,15 @@ class PyQuery { buffers_order_.push_back(name); // set nbytes and noffsets=0 here to avoid allocation; buffers set below - auto buffer_info = BufferInfo(name, 0, type, cell_val_num, 0, - 0, // TODO - is_var(name), is_nullable(name)); + auto buffer_info = BufferInfo( + name, + 0, + type, + cell_val_num, + 0, + 0, // TODO + is_var(name), + is_nullable(name)); buffer_info.data = data; buffer_info.offsets = offsets; buffers_.insert({name, buffer_info}); @@ -824,7 +860,7 @@ class PyQuery { auto size_pair = query_->est_result_size_var(name); buf_nbytes = size_pair[0]; offsets_num = size_pair[1]; - } else { // !nullable && !var + } else { // !nullable && !var buf_nbytes = query_->est_result_size(name); } @@ -860,17 +896,25 @@ class PyQuery { buffers_order_.push_back(name); buffers_.insert( - {name, BufferInfo(name, buf_nbytes, type, cell_val_num, offsets_num, - validity_num, var, nullable)}); + {name, + BufferInfo( + name, + buf_nbytes, + type, + cell_val_num, + offsets_num, + validity_num, + var, + nullable)}); } - void add_label_buffer(std::string &label_name, uint64_t ncells) { + void add_label_buffer(std::string& label_name, uint64_t ncells) { label_input_buffer_data_[label_name] = ncells; } py::object get_buffers() { py::dict rmap; - for (auto &bp : buffers_) { + for (auto& bp : buffers_) { py::list result; const auto name = bp.first; const BufferInfo b = bp.second; @@ -890,8 +934,8 @@ class PyQuery { size_t data_vals_read = b.data_vals_read; size_t validity_vals_read = b.validity_vals_read; - void *data_ptr = - (void *)((char *)b.data.data() + (data_vals_read * b.elem_nbytes)); + void* data_ptr = + (void*)((char*)b.data.data() + (data_vals_read * b.elem_nbytes)); uint64_t data_nelem = (b.data.size() - (data_vals_read * b.elem_nbytes)) / b.elem_nbytes; @@ -901,15 +945,15 @@ class PyQuery { if (b.isvar) { size_t offsets_size = b.offsets.size() - offsets_read; - uint64_t *offsets_ptr = (uint64_t *)b.offsets.data() + offsets_read; + uint64_t* offsets_ptr = (uint64_t*)b.offsets.data() + offsets_read; - query_->set_offsets_buffer(b.name, (uint64_t *)(offsets_ptr), - offsets_size); + query_->set_offsets_buffer( + b.name, (uint64_t*)(offsets_ptr), offsets_size); } if (b.isnullable) { uint64_t validity_size = b.validity.size() - validity_vals_read; - uint8_t *validity_ptr = - (uint8_t *)b.validity.data() + validity_vals_read; + uint8_t* validity_ptr = + (uint8_t*)b.validity.data() + validity_vals_read; query_->set_validity_buffer(b.name, validity_ptr, validity_size); } @@ -920,13 +964,13 @@ class PyQuery { auto result_elements = QueryExperimental::result_buffer_elements_nullable_labels(*query_); - for (const auto &read_info : result_elements) { + for (const auto& read_info : result_elements) { auto name = read_info.first; uint64_t offset_elem_num = 0, data_vals_num = 0, validity_elem_num = 0; std::tie(offset_elem_num, data_vals_num, validity_elem_num) = read_info.second; - BufferInfo &buf = buffers_.at(name); + BufferInfo& buf = buffers_.at(name); // TODO if we ever support per-attribute read offset bitsize // then need to handle here. Currently this is hard-coded to @@ -962,21 +1006,23 @@ class PyQuery { std::to_string(buf.data.size()) + ")"); } if ((Py_ssize_t)buf.offsets_read > buf.offsets.size()) { - throw TileDBError("After read query, offsets buffer out of bounds: " + - name + " (" + std::to_string(buf.offsets_read) + - " > " + std::to_string(buf.offsets.size()) + ")"); + throw TileDBError( + "After read query, offsets buffer out of bounds: " + name + " (" + + std::to_string(buf.offsets_read) + " > " + + std::to_string(buf.offsets.size()) + ")"); } if ((Py_ssize_t)buf.validity_vals_read > buf.validity.size()) { - throw TileDBError("After read query, validity buffer out of bounds: " + - name + " (" + std::to_string(buf.validity_vals_read) + - " > " + std::to_string(buf.validity.size()) + ")"); + throw TileDBError( + "After read query, validity buffer out of bounds: " + name + " (" + + std::to_string(buf.validity_vals_read) + " > " + + std::to_string(buf.validity.size()) + ")"); } } } void reset_read_elem_num() { - for (auto &bp : buffers_) { - auto &buf = bp.second; + for (auto& bp : buffers_) { + auto& buf = bp.second; buf.offsets_read = 0; buf.data_vals_read = 0; @@ -991,12 +1037,12 @@ class PyQuery { try { tmp_str = ctx_.config().get("py.max_incomplete_retries"); max_retries = std::stoull(tmp_str); - } catch (const std::invalid_argument &e) { + } catch (const std::invalid_argument& e) { (void)e; throw TileDBError( "Failed to convert 'py.max_incomplete_retries' to uint64_t ('" + tmp_str + "')"); - } catch (tiledb::TileDBError &e) { + } catch (tiledb::TileDBError& e) { (void)e; max_retries = 100; } @@ -1005,14 +1051,14 @@ class PyQuery { void resubmit_read() { tiledb_query_status_details_t status_details; - tiledb_query_get_status_details(ctx_.ptr().get(), query_.get()->ptr().get(), - &status_details); + tiledb_query_get_status_details( + ctx_.ptr().get(), query_.get()->ptr().get(), &status_details); if (status_details.incomplete_reason == TILEDB_REASON_USER_BUFFER_SIZE) { auto start_incomplete_buffer_update = std::chrono::high_resolution_clock::now(); - for (auto &bp : buffers_) { - auto &buf = bp.second; + for (auto& bp : buffers_) { + auto& buf = bp.second; // Check if values buffer should be resized if ((buf.data_vals_read == 0) || @@ -1073,17 +1119,18 @@ class PyQuery { // account for the extra element at the end of offsets in arrow mode size_t arrow_offset_size = use_arrow_ ? 1 : 0; - for (auto &bp : buffers_) { + for (auto& bp : buffers_) { auto name = bp.first; - auto &buf = bp.second; + auto& buf = bp.second; Py_ssize_t final_data_nbytes = buf.data_vals_read * buf.elem_nbytes; Py_ssize_t final_offsets_count = buf.offsets_read + arrow_offset_size; Py_ssize_t final_validity_count = buf.validity_vals_read; assert(final_data_nbytes <= buf.data.size()); - assert(final_offsets_count <= - (Py_ssize_t)(buf.offsets.size() + arrow_offset_size)); + assert( + final_offsets_count <= + (Py_ssize_t)(buf.offsets.size() + arrow_offset_size)); buf.data.resize({final_data_nbytes}); buf.offsets.resize({final_offsets_count}); @@ -1128,7 +1175,7 @@ class PyQuery { } // allocate buffers for label dimensions - for (const auto &label_data : label_input_buffer_data_) { + for (const auto& label_data : label_input_buffer_data_) { alloc_buffer(label_data.first); } @@ -1251,13 +1298,13 @@ class PyQuery { } } - py::array unpack_buffer(std::string name, py::array buf, - py::array_t off) { + py::array unpack_buffer( + std::string name, py::array buf, py::array_t off) { auto start = std::chrono::high_resolution_clock::now(); if (off.size() < 1) - TPY_ERROR_LOC(std::string("Unexpected empty offsets array ('") + name + - "')"); + TPY_ERROR_LOC( + std::string("Unexpected empty offsets array ('") + name + "')"); auto dtype = buffer_dtype(name); bool is_unicode = dtype.is(py::dtype("U")); @@ -1276,14 +1323,14 @@ class PyQuery { } auto result_array = py::array(py::dtype("O"), off.size()); - auto result_p = (py::object *)result_array.mutable_data(); + auto result_p = (py::object*)result_array.mutable_data(); uint64_t last = 0; uint64_t cur = 0; size_t size = 0; uint64_t create = 0; auto off_data = off.data(); - last = off_data[0]; // initial should always be 0 + last = off_data[0]; // initial should always be 0 for (auto i = 1; i < off.size() + 1; i++) { if (i == off.size()) cur = buf.nbytes(); @@ -1294,7 +1341,7 @@ class PyQuery { size = cur - last; py::object o; - auto data_ptr = (char *)buf.data() + last; + auto data_ptr = (char*)buf.data() + last; if (is_unicode) if (size == 0 || (data_ptr[0] == '\0' && size == 1)) { o = py::str(""); @@ -1339,7 +1386,8 @@ class PyQuery { return result_array; } - void submit_write() {} + void submit_write() { + } void submit() { if (array_->query_type() == TILEDB_READ) { @@ -1358,7 +1406,7 @@ class PyQuery { py::dict results() { py::dict results; - for (auto &buffer_name : buffers_order_) { + for (auto& buffer_name : buffers_order_) { auto bp = buffers_.at(buffer_name); results[py::str(buffer_name)] = py::make_tuple(bp.data, bp.offsets, bp.validity); @@ -1374,8 +1422,12 @@ class PyQuery { std::unique_ptr pa_pair(new PAPair()); - adapter.export_buffer(name.c_str(), &(pa_pair->array_), &(pa_pair->schema_), - nullptr, nullptr); + adapter.export_buffer( + name.c_str(), + &(pa_pair->array_), + &(pa_pair->schema_), + nullptr, + nullptr); pa_pair->exported_ = true; return pa_pair; @@ -1391,20 +1443,21 @@ class PyQuery { py::list names; py::list results; - for (auto &buffer_name : buffers_order_) { - BufferInfo &buffer_info = buffers_.at(buffer_name); + for (auto& buffer_name : buffers_order_) { + BufferInfo& buffer_info = buffers_.at(buffer_name); auto buffer_holder = new BufferHolder( buffer_info.data, buffer_info.validity, buffer_info.offsets); ArrowArray c_pa_array; ArrowSchema c_pa_schema; - adapter.export_buffer(buffer_name.c_str(), - static_cast(&c_pa_array), - static_cast(&c_pa_schema), - (tiledb::arrow::ArrowAdapter::release_cb) - BufferHolder::free_buffer_holder, - buffer_holder); + adapter.export_buffer( + buffer_name.c_str(), + static_cast(&c_pa_array), + static_cast(&c_pa_schema), + (tiledb::arrow::ArrowAdapter::release_cb) + BufferHolder::free_buffer_holder, + buffer_holder); if (is_nullable(buffer_name)) { // count zeros before converting to bitmap @@ -1421,8 +1474,8 @@ class PyQuery { // work around for SC-11522: metadata field must be set to nullptr c_pa_schema.metadata = nullptr; - py::object pa_array = pa_array_import(py::int_((ptrdiff_t)&c_pa_array), - py::int_((ptrdiff_t)&c_pa_schema)); + py::object pa_array = pa_array_import( + py::int_((ptrdiff_t)&c_pa_array), py::int_((ptrdiff_t)&c_pa_schema)); results.append(pa_array); names.append(buffer_name); @@ -1437,8 +1490,8 @@ class PyQuery { if (!query_) { throw TileDBPyError("Internal error: PyQuery not initialized!"); } - return py::cast(query_->query_status() == - tiledb::Query::Status::INCOMPLETE); + return py::cast( + query_->query_status() == tiledb::Query::Status::INCOMPLETE); } py::object estimated_result_sizes() { @@ -1470,7 +1523,7 @@ class PyQuery { py::dict results; - for (auto const &name : estim_names) { + for (auto const& name : estim_names) { size_t est_offsets = 0, est_data_bytes = 0; if (is_var(name)) { @@ -1504,9 +1557,11 @@ class PyQuery { return alloc_max_bytes_; } - std::string get_stats() { return query_->stats(); } + std::string get_stats() { + return query_->stats(); + } -private: + private: void init_config() { // get config parameters std::string tmp_str; @@ -1514,7 +1569,7 @@ class PyQuery { tmp_str = ctx_.config().get("py.init_buffer_bytes"); try { init_buffer_bytes_ = std::stoull(tmp_str); - } catch (const std::invalid_argument &e) { + } catch (const std::invalid_argument& e) { (void)e; throw std::invalid_argument( "Failed to convert 'py.init_buffer_bytes' to uint64_t ('" + @@ -1526,15 +1581,16 @@ class PyQuery { tmp_str = ctx_.config().get("py.alloc_max_bytes"); try { alloc_max_bytes_ = std::stoull(tmp_str); - } catch (const std::invalid_argument &e) { + } catch (const std::invalid_argument& e) { (void)e; throw std::invalid_argument( "Failed to convert 'py.alloc_max_bytes' to uint64_t ('" + tmp_str + "')"); } if (alloc_max_bytes_ < pow(1024, 2)) { - throw std::invalid_argument("Invalid parameter: 'py.alloc_max_bytes' " - "must be >= 1 MB (1024 ** 2 bytes)"); + throw std::invalid_argument( + "Invalid parameter: 'py.alloc_max_bytes' " + "must be >= 1 MB (1024 ** 2 bytes)"); }; } @@ -1558,9 +1614,10 @@ class PyQuery { } else if (tmp_str == "false") { exact_init_bytes_ = false; } else { - throw std::invalid_argument("Failed to convert configuration " - "'py.exact_init_buffer_bytes' to bool ('" + - tmp_str + "')"); + throw std::invalid_argument( + "Failed to convert configuration " + "'py.exact_init_buffer_bytes' to bool ('" + + tmp_str + "')"); } } @@ -1578,7 +1635,7 @@ class PyQuery { } } -}; // namespace tiledbpy +}; // namespace tiledbpy void init_stats() { g_stats.reset(new StatsInfo()); @@ -1592,27 +1649,31 @@ void init_stats() { stats_counters["py.query_retries_count"] = TimerType(); } -void disable_stats() { g_stats.reset(nullptr); } +void disable_stats() { + g_stats.reset(nullptr); +} void increment_stat(std::string key, double value) { - auto &stats_counters = g_stats.get()->counters; + auto& stats_counters = g_stats.get()->counters; if (stats_counters.count(key) == 0) stats_counters[key] = TimerType(); - auto &timer = stats_counters[key]; + auto& timer = stats_counters[key]; auto incr = std::chrono::duration(value); timer += incr; } -bool use_stats() { return (bool)g_stats; } +bool use_stats() { + return (bool)g_stats; +} py::object get_stats() { if (!g_stats) { TPY_ERROR_LOC("Stats counters are not uninitialized!") } - auto &stats_counters = g_stats.get()->counters; + auto& stats_counters = g_stats.get()->counters; py::dict res; for (auto iter = stats_counters.begin(); iter != stats_counters.end(); @@ -1640,7 +1701,7 @@ py::object python_internal_stats(bool dict = false) { return std::move(stats); } - for (auto &stat : counters) { + for (auto& stat : counters) { stats[py::str(stat.first)] = stat.second.count(); } @@ -1657,7 +1718,7 @@ py::object python_internal_stats(bool dict = false) { os << std::endl; os << "==== Python Stats ====" << std::endl << std::endl; - for (auto &stat : counters) { + for (auto& stat : counters) { os << " " << stat.first << " : " << stat.second.count() << std::endl; } @@ -1666,12 +1727,12 @@ py::object python_internal_stats(bool dict = false) { } py::str as_built_dump() { - tiledb_string_t *s; + tiledb_string_t* s; int rc = tiledb_as_built_dump(&s); if (rc != TILEDB_OK) { TPY_ERROR_LOC("Could not dump as built."); } - const char *data_ptr; + const char* data_ptr; py::size_t length; tiledb_string_view(s, &data_ptr, &length); @@ -1681,13 +1742,18 @@ py::str as_built_dump() { return res; } -void init_core(py::module &m) { +void init_core(py::module& m) { init_query_condition(m); auto pq = py::class_(m, "PyQuery") - .def(py::init()) + .def(py::init< + const Context&, + py::object, + py::iterable, + py::iterable, + py::object, + py::object>()) .def("add_label_buffer", &PyQuery::add_label_buffer) .def("buffer_dtype", &PyQuery::buffer_dtype) .def("results", &PyQuery::results) @@ -1705,21 +1771,26 @@ void init_core(py::module &m) { .def("_buffer_to_pa", &PyQuery::buffer_to_pa) .def("_buffers_to_pa_table", &PyQuery::buffers_to_pa_table) .def("_test_array", &PyQuery::_test_array) - .def("_test_err", - [](py::object self, std::string s) { throw TileDBPyError(s); }) + .def( + "_test_err", + [](py::object self, std::string s) { throw TileDBPyError(s); }) .def_readwrite("_preload_metadata", &PyQuery::preload_metadata_) .def_readwrite("_return_incomplete", &PyQuery::return_incomplete_) // properties .def_property_readonly("is_incomplete", &PyQuery::is_incomplete) - .def_property_readonly("_test_init_buffer_bytes", - &PyQuery::_test_init_buffer_bytes) - .def_property_readonly("_test_alloc_max_bytes", - &PyQuery::_test_alloc_max_bytes) + .def_property_readonly( + "_test_init_buffer_bytes", &PyQuery::_test_init_buffer_bytes) + .def_property_readonly( + "_test_alloc_max_bytes", &PyQuery::_test_alloc_max_bytes) .def_readonly("retries", &PyQuery::retries_); py::class_(m, "PyAgg") - .def(py::init(), - "ctx"_a, "py_array"_a, "py_layout"_a, "attr_to_aggs_input"_a) + .def( + py::init(), + "ctx"_a, + "py_array"_a, + "py_layout"_a, + "attr_to_aggs_input"_a) .def("set_subarray", &PyAgg::set_subarray) .def("set_cond", &PyAgg::set_cond) .def("get_aggregate", &PyAgg::get_aggregate); @@ -1733,8 +1804,8 @@ void init_core(py::module &m) { m.def("init_stats", &init_stats); m.def("disable_stats", &disable_stats); - m.def("python_internal_stats", &python_internal_stats, - py::arg("dict") = false); + m.def( + "python_internal_stats", &python_internal_stats, py::arg("dict") = false); m.def("increment_stat", &increment_stat); m.def("get_stats", &get_stats); m.def("use_stats", &use_stats); @@ -1752,11 +1823,11 @@ void init_core(py::module &m) { try { if (p) std::rethrow_exception(p); - } catch (const TileDBPyError &e) { + } catch (const TileDBPyError& e) { PyErr_SetString(tiledb_py_error.ptr(), e.what()); - } catch (const tiledb::TileDBError &e) { + } catch (const tiledb::TileDBError& e) { PyErr_SetString(tiledb_py_error.ptr(), e.what()); - } catch (py::builtin_exception &e) { + } catch (py::builtin_exception& e) { // just forward the error throw; //} catch (std::runtime_error &e) { @@ -1764,4 +1835,4 @@ void init_core(py::module &m) { } }); }; -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/debug.cc b/tiledb/debug.cc index 82a627c5d1..28ef9a3437 100644 --- a/tiledb/debug.cc +++ b/tiledb/debug.cc @@ -28,8 +28,8 @@ __attribute__((used)) static std::string pyrepr(py::object o) { return py::cast(py::eval("repr(_v)", py::globals(), locals)); } -__attribute__((used)) static void pycall1(const char *expr, - pybind11::object o = py::none()) { +__attribute__((used)) static void pycall1( + const char* expr, pybind11::object o = py::none()) { // this doesn't work in lldb // py::scoped_interpreter guard{}; @@ -53,16 +53,16 @@ __attribute__((used)) static void pycall1(const char *expr, if (!res.is(py::none())) { py::print(res); } - } catch (py::error_already_set &e) { + } catch (py::error_already_set& e) { std::cout << "pycall error_already_set: " << std::endl; - } catch (std::runtime_error &e) { + } catch (std::runtime_error& e) { std::cout << "pycall runtime_error: " << e.what() << std::endl; } catch (...) { std::cout << "pycall unknown exception" << std::endl; } } -__attribute__((used)) static void pycall(const char *expr) { +__attribute__((used)) static void pycall(const char* expr) { pycall1(expr, py::none()); } @@ -70,6 +70,6 @@ __attribute__((used)) static void pyerror() { // print the last py error, if any } } -}; // namespace +}; // namespace #endif diff --git a/tiledb/fragment.cc b/tiledb/fragment.cc index e463626b50..aae76da82b 100644 --- a/tiledb/fragment.cc +++ b/tiledb/fragment.cc @@ -6,8 +6,8 @@ #include +#include // C++ #include "util.h" -#include // C++ #if !defined(NDEBUG) // #include "debug.cc" @@ -21,8 +21,7 @@ namespace py = pybind11; using namespace pybind11::literals; class PyFragmentInfo { - -private: + private: Context ctx_; unique_ptr fi_; @@ -41,17 +40,20 @@ class PyFragmentInfo { py::tuple mbrs_; py::tuple array_schema_name_; -public: - tiledb_ctx_t *c_ctx_; + public: + tiledb_ctx_t* c_ctx_; -public: + public: PyFragmentInfo() = delete; - PyFragmentInfo(const string &uri, py::object schema, py::bool_ include_mbrs, - py::object ctx) { + PyFragmentInfo( + const string& uri, + py::object schema, + py::bool_ include_mbrs, + py::object ctx) { schema_ = schema; - tiledb_ctx_t *c_ctx_ = (py::capsule)ctx.attr("__capsule__")(); + tiledb_ctx_t* c_ctx_ = (py::capsule)ctx.attr("__capsule__")(); if (c_ctx_ == nullptr) TPY_ERROR_LOC("Invalid context pointer!"); @@ -80,26 +82,48 @@ class PyFragmentInfo { close(); } - uint32_t get_num_fragments() { return num_fragments_; }; - py::tuple get_uri() { return uri_; }; - py::tuple get_version() { return version_; }; - py::tuple get_nonempty_domain() { return nonempty_domain_; }; - py::tuple get_cell_num() { return cell_num_; }; - py::tuple get_timestamp_range() { return timestamp_range_; }; - py::tuple get_sparse() { return sparse_; }; + uint32_t get_num_fragments() { + return num_fragments_; + }; + py::tuple get_uri() { + return uri_; + }; + py::tuple get_version() { + return version_; + }; + py::tuple get_nonempty_domain() { + return nonempty_domain_; + }; + py::tuple get_cell_num() { + return cell_num_; + }; + py::tuple get_timestamp_range() { + return timestamp_range_; + }; + py::tuple get_sparse() { + return sparse_; + }; uint32_t get_unconsolidated_metadata_num() { return unconsolidated_metadata_num_; }; py::tuple get_has_consolidated_metadata() { return has_consolidated_metadata_; }; - py::tuple get_to_vacuum() { return to_vacuum_; }; - py::tuple get_mbrs() { return mbrs_; }; - py::tuple get_array_schema_name() { return array_schema_name_; }; + py::tuple get_to_vacuum() { + return to_vacuum_; + }; + py::tuple get_mbrs() { + return mbrs_; + }; + py::tuple get_array_schema_name() { + return array_schema_name_; + }; - void dump() const { return fi_->dump(stdout); } + void dump() const { + return fi_->dump(stdout); + } -private: + private: template py::object for_all_fid(T (FragmentInfo::*fn)(uint32_t) const) const { py::list l; @@ -113,12 +137,14 @@ class PyFragmentInfo { void load() const { try { fi_->load(); - } catch (TileDBError &e) { + } catch (TileDBError& e) { TPY_ERROR_LOC(e.what()); } } - void close() { fi_.reset(); } + void close() { + fi_.reset(); + } py::tuple fill_uri() const { return for_all_fid(&FragmentInfo::fragment_uri); @@ -166,9 +192,10 @@ class PyFragmentInfo { auto datetime64 = np.attr("datetime64"); auto datetime_data = np.attr("datetime_data"); - uint64_t *dates = static_cast(buffer.ptr); - limits = py::make_tuple(datetime64(dates[0], datetime_data(type)), - datetime64(dates[1], datetime_data(type))); + uint64_t* dates = static_cast(buffer.ptr); + limits = py::make_tuple( + datetime64(dates[0], datetime_data(type)), + datetime64(dates[1], datetime_data(type))); } return std::move(limits); @@ -202,15 +229,21 @@ class PyFragmentInfo { return for_all_fid(&FragmentInfo::timestamp_range); } - uint32_t fragment_num() const { return fi_->fragment_num(); } + uint32_t fragment_num() const { + return fi_->fragment_num(); + } - py::tuple fill_sparse() const { return for_all_fid(&FragmentInfo::sparse); } + py::tuple fill_sparse() const { + return for_all_fid(&FragmentInfo::sparse); + } py::tuple fill_cell_num() const { return for_all_fid(&FragmentInfo::cell_num); } - py::tuple fill_version() const { return for_all_fid(&FragmentInfo::version); } + py::tuple fill_version() const { + return for_all_fid(&FragmentInfo::version); + } py::tuple fill_has_consolidated_metadata() const { return for_all_fid(&FragmentInfo::has_consolidated_metadata); @@ -220,7 +253,9 @@ class PyFragmentInfo { return fi_->unconsolidated_metadata_num(); } - uint32_t to_vacuum_num() const { return fi_->to_vacuum_num(); } + uint32_t to_vacuum_num() const { + return fi_->to_vacuum_num(); + } py::tuple fill_to_vacuum_uri() const { py::list l; @@ -283,9 +318,9 @@ class PyFragmentInfo { } }; -void init_fragment(py::module &m) { +void init_fragment(py::module& m) { py::class_(m, "PyFragmentInfo") - .def(py::init()) + .def(py::init()) .def("get_num_fragments", &PyFragmentInfo::get_num_fragments) .def("get_uri", &PyFragmentInfo::get_uri) .def("get_version", &PyFragmentInfo::get_version) @@ -293,14 +328,16 @@ void init_fragment(py::module &m) { .def("get_cell_num", &PyFragmentInfo::get_cell_num) .def("get_timestamp_range", &PyFragmentInfo::get_timestamp_range) .def("get_sparse", &PyFragmentInfo::get_sparse) - .def("get_unconsolidated_metadata_num", - &PyFragmentInfo::get_unconsolidated_metadata_num) - .def("get_has_consolidated_metadata", - &PyFragmentInfo::get_has_consolidated_metadata) + .def( + "get_unconsolidated_metadata_num", + &PyFragmentInfo::get_unconsolidated_metadata_num) + .def( + "get_has_consolidated_metadata", + &PyFragmentInfo::get_has_consolidated_metadata) .def("get_to_vacuum", &PyFragmentInfo::get_to_vacuum) .def("get_mbrs", &PyFragmentInfo::get_mbrs) .def("get_array_schema_name", &PyFragmentInfo::get_array_schema_name) .def("dump", &PyFragmentInfo::dump); } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/main.cc b/tiledb/main.cc index 8f06efecd5..3f363539f4 100644 --- a/tiledb/main.cc +++ b/tiledb/main.cc @@ -4,17 +4,17 @@ namespace tiledbpy { namespace py = pybind11; -void init_core(py::module &); +void init_core(py::module&); // void _debug(py::module &); -void init_fragment(py::module &); +void init_fragment(py::module&); // void init_query_condition(py::module &); -void init_schema_evolution(py::module &); +void init_schema_evolution(py::module&); #if defined(TILEDB_SERIALIZATION) -void init_serialization(py::module &); -void init_test_serialization(py::module &); +void init_serialization(py::module&); +void init_test_serialization(py::module&); #endif -void init_test_metadata(py::module &); -void init_test_webp_filter(py::module &); +void init_test_metadata(py::module&); +void init_test_webp_filter(py::module&); PYBIND11_MODULE(main, m) { init_core(m); @@ -30,4 +30,4 @@ PYBIND11_MODULE(main, m) { init_test_webp_filter(m); } -} // namespace tiledbpy +} // namespace tiledbpy diff --git a/tiledb/npbuffer.cc b/tiledb/npbuffer.cc index 0eec24649d..539bcda527 100644 --- a/tiledb/npbuffer.cc +++ b/tiledb/npbuffer.cc @@ -6,16 +6,16 @@ #include #include -#include "util.h" #include #include #include +#include "util.h" #if !defined(NDEBUG) // #include "debug.cc" #endif -#include // C++ +#include // C++ // anonymous namespace for helper functions namespace { @@ -30,8 +30,9 @@ bool issubdtype(py::dtype t1, py::dtype t2) { return py::cast(npsubdtype(t1, t2)); } -template py::dtype get_dtype(T obj) { - auto &api = py::detail::npy_api::get(); +template +py::dtype get_dtype(T obj) { + auto& api = py::detail::npy_api::get(); if (api.PyArray_Check_(obj.ptr())) { return py::cast(obj).dtype(); @@ -45,12 +46,12 @@ template py::dtype get_dtype(T obj) { // note: d1::dtype.is(d2) checks *object identity* which is // not what we want. bool dtype_equal(py::dtype d1, py::dtype d2) { - auto &api = py::detail::npy_api::get(); + auto& api = py::detail::npy_api::get(); return api.PyArray_EquivTypes_(d1.ptr(), d2.ptr()); } -}; // namespace +}; // namespace namespace tiledbpy { @@ -61,7 +62,7 @@ using namespace pybind11::literals; #if PY_MAJOR_VERSION >= 3 class NumpyConvert { -private: + private: bool use_iter_ = false; bool allow_unicode_ = true; size_t data_nbytes_ = 0; @@ -70,8 +71,8 @@ class NumpyConvert { py::array input_; // we are using vector as a buffer here because they are grown in some // situations - std::vector *data_buf_; - std::vector *offset_buf_; + std::vector* data_buf_; + std::vector* offset_buf_; void convert_unicode() { // Convert array of strings to UTF-8 buffer+offsets @@ -94,9 +95,9 @@ class NumpyConvert { // size (bytes) of current object data Py_ssize_t sz = 0; // object data (string or bytes) - const char *input_p = nullptr; + const char* input_p = nullptr; - unsigned char *output_p = nullptr; + unsigned char* output_p = nullptr; output_p = data_buf_->data(); // avoid one interpreter roundtrip @@ -117,11 +118,11 @@ class NumpyConvert { if (PyUnicode_Check(u.ptr())) { // TODO see if we can do this with PyUnicode_AsUTF8String u_encoded = npstrencode(u); - rc = PyBytes_AsStringAndSize(u_encoded.ptr(), - const_cast(&input_p), &sz); + rc = PyBytes_AsStringAndSize( + u_encoded.ptr(), const_cast(&input_p), &sz); } else { - rc = PyBytes_AsStringAndSize(u.ptr(), const_cast(&input_p), - &sz); + rc = + PyBytes_AsStringAndSize(u.ptr(), const_cast(&input_p), &sz); } if (rc == -1) { @@ -151,7 +152,7 @@ class NumpyConvert { void convert_bytes() { // Convert array of bytes objects or ASCII strings to buffer+offsets - assert(input_.itemsize() > 0); // must have fixed-length array + assert(input_.itemsize() > 0); // must have fixed-length array // we know exact offset count offset_buf_->resize(input_len_); @@ -162,9 +163,9 @@ class NumpyConvert { // size (bytes) of current object data Py_ssize_t sz = 0; // object data (string or bytes) - const char *input_p = nullptr; + const char* input_p = nullptr; - unsigned char *output_p = nullptr; + unsigned char* output_p = nullptr; output_p = data_buf_->data(); int rc; @@ -190,7 +191,7 @@ if (PyUnicode_Check(u.ptr())) { } */ - rc = PyBytes_AsStringAndSize(o, const_cast(&input_p), &sz); + rc = PyBytes_AsStringAndSize(o, const_cast(&input_p), &sz); if (rc == -1) { throw std::runtime_error( "PyBytes_AsStringAndSize failed to encode string"); @@ -216,7 +217,7 @@ if (PyUnicode_Check(u.ptr())) { void convert_object() { // Convert np.dtype("O") array of objects to buffer+offsets - auto &api = py::detail::npy_api::get(); + auto& api = py::detail::npy_api::get(); offset_buf_->resize(input_len_); @@ -225,7 +226,7 @@ if (PyUnicode_Check(u.ptr())) { // size (bytes) of current object data Py_ssize_t sz = 0; // current data - const char *input_p = nullptr; + const char* input_p = nullptr; auto input_size = input_.size(); py::dtype first_dtype; @@ -234,7 +235,7 @@ if (PyUnicode_Check(u.ptr())) { for (int64_t idx = 0; idx < input_size; idx++) { offset_buf_->data()[idx] = data_nbytes_; - PyObject *o = input_unchecked.data(idx)->ptr(); + PyObject* o = input_unchecked.data(idx)->ptr(); assert(o != nullptr); // NOTE: every branch below *must* initialize first_dtype @@ -258,7 +259,7 @@ if (PyUnicode_Check(u.ptr())) { } else if (PyBytes_Check(o)) { // ASCII only auto res = - PyBytes_AsStringAndSize(o, const_cast(&input_p), &sz); + PyBytes_AsStringAndSize(o, const_cast(&input_p), &sz); if (idx < 1) first_dtype = py::dtype("bytes"); @@ -286,7 +287,7 @@ if (PyUnicode_Check(u.ptr())) { auto a = py::cast(o); sz = sizeof(bool); bool bool_value = a; - input_p = reinterpret_cast(&bool_value); + input_p = reinterpret_cast(&bool_value); } else { // TODO write the type in the error here // auto o_h = py::reinterpret_borrow(o); @@ -302,11 +303,11 @@ if (PyUnicode_Check(u.ptr())) { data_buf_->resize(data_nbytes_); // second pass: copy the data to output buffer - unsigned char *output_p = data_buf_->data(); + unsigned char* output_p = data_buf_->data(); // copy data to output buffers for (int64_t idx = 0; idx < input_size; idx++) { - PyObject *pyobj_p = input_unchecked.data(idx)->ptr(); + PyObject* pyobj_p = input_unchecked.data(idx)->ptr(); assert(pyobj_p != nullptr); @@ -315,16 +316,16 @@ if (PyUnicode_Check(u.ptr())) { assert(input_p != nullptr); } else if (PyBytes_Check(pyobj_p)) { // TODO error check? - PyBytes_AsStringAndSize(pyobj_p, const_cast(&input_p), &sz); + PyBytes_AsStringAndSize(pyobj_p, const_cast(&input_p), &sz); } else if (api.PyArray_Check_(pyobj_p)) { auto arr = py::cast(pyobj_p); sz = arr.nbytes(); - input_p = (const char *)arr.data(); + input_p = (const char*)arr.data(); } else if (PyBool_Check(pyobj_p)) { py::bool_ bool_obj = py::cast(pyobj_p); sz = sizeof(bool); bool bool_value = bool_obj; - input_p = reinterpret_cast(&bool_value); + input_p = reinterpret_cast(&bool_value); } else { // TODO add object type TPY_ERROR_LOC("Unexpected object type in buffer conversion"); @@ -342,7 +343,7 @@ if (PyUnicode_Check(u.ptr())) { // For non-contiguous arrays (such as views) we must iterate rather // than indexing directly. - auto &npy_api = py::detail::npy_api::get(); + auto& npy_api = py::detail::npy_api::get(); offset_buf_->resize(input_.size()); @@ -351,7 +352,7 @@ if (PyUnicode_Check(u.ptr())) { // size (bytes) of current object data Py_ssize_t sz = 0; // current data - const char *input_p = nullptr; + const char* input_p = nullptr; size_t idx = 0; @@ -364,7 +365,7 @@ if (PyUnicode_Check(u.ptr())) { } offset_buf_->data()[idx] = data_nbytes_; - PyObject *obj_p = obj_h.ptr(); + PyObject* obj_p = obj_h.ptr(); // we must check each dtype because object arrays are not guaranteed to // be homogenous @@ -394,7 +395,7 @@ if (PyUnicode_Check(u.ptr())) { } else if (PyBytes_Check(obj_p)) { // ASCII only auto res = - PyBytes_AsStringAndSize(obj_p, const_cast(&input_p), &sz); + PyBytes_AsStringAndSize(obj_p, const_cast(&input_p), &sz); if (res == -1) { // TODO TPY_ERROR_LOC @@ -411,7 +412,7 @@ if (PyUnicode_Check(u.ptr())) { py::bool_ bool_obj = py::cast(obj_p); sz = sizeof(bool); bool bool_value = bool_obj; - input_p = reinterpret_cast(&bool_value); + input_p = reinterpret_cast(&bool_value); } else { auto errmsg = std::string("Unexpected object type in string conversion"); @@ -423,7 +424,7 @@ if (PyUnicode_Check(u.ptr())) { data_buf_->resize(data_nbytes_); // second pass: write the data to output buffer - unsigned char *output_p = data_buf_->data(); + unsigned char* output_p = data_buf_->data(); // reset the iterator iter = input_.attr("flat"); @@ -437,19 +438,19 @@ if (PyUnicode_Check(u.ptr())) { assert(input_p != nullptr); } else if (PyBytes_Check(obj_p)) { // TODO error check? - PyBytes_AsStringAndSize(obj_p, const_cast(&input_p), &sz); + PyBytes_AsStringAndSize(obj_p, const_cast(&input_p), &sz); } else if (npy_api.PyArray_Check_(obj_p)) { // auto pao = (PyArrayObject*)o; // input_p = (const char*)PyArray_DATA(pao); // sz = PyArray_NBYTES(pao); auto o_a = py::cast(obj_h); sz = o_a.nbytes(); - input_p = (const char *)o_a.data(); + input_p = (const char*)o_a.data(); } else if (PyBool_Check(obj_p)) { py::bool_ bool_obj = py::cast(obj_p); sz = sizeof(bool); bool bool_value = bool_obj; - input_p = reinterpret_cast(&bool_value); + input_p = reinterpret_cast(&bool_value); } else { TPY_ERROR_LOC("Unexpected object type in buffer conversion"); } @@ -460,7 +461,7 @@ if (PyUnicode_Check(u.ptr())) { } } -public: + public: /* Initialize the converter */ @@ -473,13 +474,13 @@ if (PyUnicode_Check(u.ptr())) { // which is what we want try { v.attr("shape") = py::int_(input.size()); - } catch (py::error_already_set &e) { + } catch (py::error_already_set& e) { if (e.matches(PyExc_AttributeError)) { use_iter_ = true; } else { throw; } - } catch (std::exception &e) { + } catch (std::exception& e) { std::cout << e.what() << std::endl; } input_ = v; @@ -503,8 +504,12 @@ if (PyUnicode_Check(u.ptr())) { /* Set allow_unicode_ flag */ - bool allow_unicode() { return allow_unicode_; } - void allow_unicode(bool allow_unicode) { allow_unicode_ = allow_unicode; } + bool allow_unicode() { + return allow_unicode_; + } + void allow_unicode(bool allow_unicode) { + allow_unicode_ = allow_unicode; + } /* Returns a tuple of py::array containing @@ -532,19 +537,19 @@ if (PyUnicode_Check(u.ptr())) { } auto tmp_data_buf_p = data_buf_; - auto data_ref = py::capsule(data_buf_, [](void *v) { - delete reinterpret_cast *>(v); + auto data_ref = py::capsule(data_buf_, [](void* v) { + delete reinterpret_cast*>(v); }); - data_buf_ = nullptr; // disown: capsule owns it + data_buf_ = nullptr; // disown: capsule owns it auto tmp_offset_buf_p = offset_buf_; - auto offset_ref = py::capsule(offset_buf_, [](void *v) { - delete reinterpret_cast *>(v); + auto offset_ref = py::capsule(offset_buf_, [](void* v) { + delete reinterpret_cast*>(v); }); - offset_buf_ = nullptr; // disown: capsule owns it now + offset_buf_ = nullptr; // disown: capsule owns it now - auto data_np = py::array_t(tmp_data_buf_p->size(), - tmp_data_buf_p->data(), data_ref); + auto data_np = py::array_t( + tmp_data_buf_p->size(), tmp_data_buf_p->data(), data_ref); auto offset_np = py::array_t( tmp_offset_buf_p->size(), tmp_offset_buf_p->data(), offset_ref); @@ -553,8 +558,8 @@ if (PyUnicode_Check(u.ptr())) { }; #endif -py::tuple convert_np(py::array input, bool allow_unicode, - bool use_fallback = false) { +py::tuple convert_np( + py::array input, bool allow_unicode, bool use_fallback = false) { #if PY_MAJOR_VERSION >= 3 if (use_fallback) { #endif @@ -571,4 +576,4 @@ py::tuple convert_np(py::array input, bool allow_unicode, #endif } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/npbuffer.h b/tiledb/npbuffer.h index adf24cbc0e..9958d10994 100644 --- a/tiledb/npbuffer.h +++ b/tiledb/npbuffer.h @@ -1,14 +1,14 @@ -#include "util.h" #include #include #include +#include "util.h" namespace tiledbpy { namespace py = pybind11; using namespace pybind11::literals; -py::tuple convert_np(py::array input, bool allow_unicode = true, - bool use_fallback = false); +py::tuple convert_np( + py::array input, bool allow_unicode = true, bool use_fallback = false); -} // namespace tiledbpy \ No newline at end of file +} // namespace tiledbpy diff --git a/tiledb/numpyFlags.h b/tiledb/numpyFlags.h index be21064176..4d18a85f8a 100644 --- a/tiledb/numpyFlags.h +++ b/tiledb/numpyFlags.h @@ -1,2 +1,2 @@ // Turn off cython generated deprecation warnings -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION \ No newline at end of file +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION diff --git a/tiledb/py_arrow_io_impl.h b/tiledb/py_arrow_io_impl.h index f2e8a60dda..9ccf917fc2 100644 --- a/tiledb/py_arrow_io_impl.h +++ b/tiledb/py_arrow_io_impl.h @@ -46,18 +46,18 @@ struct ArrowSchema { // Array type description - const char *format; - const char *name; - const char *metadata; + const char* format; + const char* name; + const char* metadata; int64_t flags; int64_t n_children; - struct ArrowSchema **children; - struct ArrowSchema *dictionary; + struct ArrowSchema** children; + struct ArrowSchema* dictionary; // Release callback - void (*release)(struct ArrowSchema *); + void (*release)(struct ArrowSchema*); // Opaque producer-specific data - void *private_data; + void* private_data; }; struct ArrowArray { @@ -67,14 +67,14 @@ struct ArrowArray { int64_t offset; int64_t n_buffers; int64_t n_children; - const void **buffers; - struct ArrowArray **children; - struct ArrowArray *dictionary; + const void** buffers; + struct ArrowArray** children; + struct ArrowArray* dictionary; // Release callback - void (*release)(struct ArrowArray *); + void (*release)(struct ArrowArray*); // Opaque producer-specific data - void *private_data; + void* private_data; }; /* End Arrow C API */ /* ************************************************************************ */ @@ -91,9 +91,10 @@ struct ArrowArray { using _TileDBError = tiledb::TileDBError; #ifndef NDEBUG -#define TDB_LERROR(m) \ - _TileDBError(std::string(m) + " (" + __FILE__ + ":" + \ - std::to_string(__LINE__) + ")"); +#define TDB_LERROR(m) \ + _TileDBError( \ + std::string(m) + " (" + __FILE__ + ":" + std::to_string(__LINE__) + \ + ")"); #else #define TDB_LERROR tiledb::TileDBError #endif @@ -107,8 +108,9 @@ namespace arrow { // Arrow format and representation struct ArrowInfo { - ArrowInfo(std::string fmt, const std::string &rep = std::string()) - : fmt_(fmt), rep_(rep){}; + ArrowInfo(std::string fmt, const std::string& rep = std::string()) + : fmt_(fmt) + , rep_(rep) {}; std::string fmt_; std::string rep_; @@ -126,13 +128,13 @@ struct TypeInfo { struct BufferInfo { TypeInfo tdbtype; - bool is_var; // is var-length - uint64_t data_num; // number of data elements - void *data; // data pointer - uint64_t data_elem_size; // bytes per data element - uint64_t offsets_num; // number of offsets - void *offsets; // offsets pointer - size_t offsets_elem_size; // bytes per offset element + bool is_var; // is var-length + uint64_t data_num; // number of data elements + void* data; // data pointer + uint64_t data_elem_size; // bytes per data element + uint64_t offsets_num; // number of offsets + void* offsets; // offsets pointer + size_t offsets_elem_size; // bytes per offset element }; /* ****************************** */ @@ -174,98 +176,99 @@ ArrowInfo tiledb_buffer_arrow_fmt(BufferInfo bufferinfo, bool use_list = true) { */ switch (typeinfo.type) { - //////////////////////////////////////////////////////////////////////// - case TILEDB_STRING_ASCII: - case TILEDB_STRING_UTF8: - if (bufferinfo.offsets_elem_size == 4) { - return ArrowInfo("u"); - } else { - return ArrowInfo("U"); - } - case TILEDB_CHAR: - if (!bufferinfo.is_var) { - return ArrowInfo("w:" + std::to_string(cell_val_num)); - } else if (bufferinfo.offsets_elem_size == 4) { - return ArrowInfo("z"); - } else { - return ArrowInfo("Z"); - } - case TILEDB_INT32: - return ArrowInfo("i"); - case TILEDB_INT64: - return ArrowInfo("l"); - case TILEDB_FLOAT32: - return ArrowInfo("f"); - case TILEDB_FLOAT64: - return ArrowInfo("g"); - case TILEDB_BLOB: + //////////////////////////////////////////////////////////////////////// + case TILEDB_STRING_ASCII: + case TILEDB_STRING_UTF8: + if (bufferinfo.offsets_elem_size == 4) { + return ArrowInfo("u"); + } else { + return ArrowInfo("U"); + } + case TILEDB_CHAR: + if (!bufferinfo.is_var) { + return ArrowInfo("w:" + std::to_string(cell_val_num)); + } else if (bufferinfo.offsets_elem_size == 4) { + return ArrowInfo("z"); + } else { + return ArrowInfo("Z"); + } + case TILEDB_INT32: + return ArrowInfo("i"); + case TILEDB_INT64: + return ArrowInfo("l"); + case TILEDB_FLOAT32: + return ArrowInfo("f"); + case TILEDB_FLOAT64: + return ArrowInfo("g"); + case TILEDB_BLOB: #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 21 - case TILEDB_GEOM_WKB: - case TILEDB_GEOM_WKT: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: #endif - return ArrowInfo("B"); - case TILEDB_INT8: - return ArrowInfo("c"); - case TILEDB_UINT8: - return ArrowInfo("C"); - case TILEDB_INT16: - return ArrowInfo("s"); - case TILEDB_UINT16: - return ArrowInfo("S"); - case TILEDB_UINT32: - return ArrowInfo("I"); - case TILEDB_UINT64: - return ArrowInfo("L"); - - case TILEDB_TIME_SEC: - return ArrowInfo("tts"); - case TILEDB_TIME_MS: - return ArrowInfo("ttm"); - case TILEDB_TIME_US: - return ArrowInfo("ttu"); - case TILEDB_TIME_NS: - return ArrowInfo("ttn"); - case TILEDB_DATETIME_SEC: - return ArrowInfo("tss:"); - case TILEDB_DATETIME_MS: - return ArrowInfo("tsm:"); - case TILEDB_DATETIME_US: - return ArrowInfo("tsu:"); - case TILEDB_DATETIME_NS: - return ArrowInfo("tsn:"); - // TILEDB_BOOL is stored as a uint8_t but arrow::Type::BOOL is 1 bit - case TILEDB_BOOL: - return ArrowInfo("C"); - - // TODO: these could potentially be rep'd w/ additional - // language-specific metadata - case TILEDB_DATETIME_YEAR: - case TILEDB_DATETIME_MONTH: - case TILEDB_DATETIME_WEEK: - case TILEDB_DATETIME_DAY: - case TILEDB_DATETIME_HR: - case TILEDB_DATETIME_MIN: - case TILEDB_DATETIME_PS: - case TILEDB_DATETIME_FS: - case TILEDB_DATETIME_AS: - case TILEDB_TIME_MIN: - case TILEDB_TIME_PS: - case TILEDB_TIME_FS: - case TILEDB_TIME_AS: - case TILEDB_STRING_UTF16: - case TILEDB_STRING_UTF32: - case TILEDB_STRING_UCS2: - case TILEDB_STRING_UCS4: - case TILEDB_ANY: - default: - break; + return ArrowInfo("B"); + case TILEDB_INT8: + return ArrowInfo("c"); + case TILEDB_UINT8: + return ArrowInfo("C"); + case TILEDB_INT16: + return ArrowInfo("s"); + case TILEDB_UINT16: + return ArrowInfo("S"); + case TILEDB_UINT32: + return ArrowInfo("I"); + case TILEDB_UINT64: + return ArrowInfo("L"); + + case TILEDB_TIME_SEC: + return ArrowInfo("tts"); + case TILEDB_TIME_MS: + return ArrowInfo("ttm"); + case TILEDB_TIME_US: + return ArrowInfo("ttu"); + case TILEDB_TIME_NS: + return ArrowInfo("ttn"); + case TILEDB_DATETIME_SEC: + return ArrowInfo("tss:"); + case TILEDB_DATETIME_MS: + return ArrowInfo("tsm:"); + case TILEDB_DATETIME_US: + return ArrowInfo("tsu:"); + case TILEDB_DATETIME_NS: + return ArrowInfo("tsn:"); + // TILEDB_BOOL is stored as a uint8_t but arrow::Type::BOOL is 1 bit + case TILEDB_BOOL: + return ArrowInfo("C"); + + // TODO: these could potentially be rep'd w/ additional + // language-specific metadata + case TILEDB_DATETIME_YEAR: + case TILEDB_DATETIME_MONTH: + case TILEDB_DATETIME_WEEK: + case TILEDB_DATETIME_DAY: + case TILEDB_DATETIME_HR: + case TILEDB_DATETIME_MIN: + case TILEDB_DATETIME_PS: + case TILEDB_DATETIME_FS: + case TILEDB_DATETIME_AS: + case TILEDB_TIME_MIN: + case TILEDB_TIME_PS: + case TILEDB_TIME_FS: + case TILEDB_TIME_AS: + case TILEDB_STRING_UTF16: + case TILEDB_STRING_UTF32: + case TILEDB_STRING_UCS2: + case TILEDB_STRING_UCS4: + case TILEDB_ANY: + default: + break; } - throw TDB_LERROR("TileDB-Arrow: tiledb datatype not understood ('" + - tiledb::impl::type_to_str(typeinfo.type) + - "', cell_val_num: " + std::to_string(cell_val_num) + ")"); + throw TDB_LERROR( + "TileDB-Arrow: tiledb datatype not understood ('" + + tiledb::impl::type_to_str(typeinfo.type) + + "', cell_val_num: " + std::to_string(cell_val_num) + ")"); } -TypeInfo arrow_type_to_tiledb(ArrowSchema *arw_schema) { +TypeInfo arrow_type_to_tiledb(ArrowSchema* arw_schema) { auto fmt = std::string(arw_schema->format); bool large = false; if (fmt == "+l") { @@ -320,7 +323,7 @@ TypeInfo arrow_type_to_tiledb(ArrowSchema *arw_schema) { "'"); } -TypeInfo tiledb_dt_info(const ArraySchema &schema, const std::string &name) { +TypeInfo tiledb_dt_info(const ArraySchema& schema, const std::string& name) { if (schema.has_attribute(name)) { auto attr = schema.attribute(name); auto retval = TypeInfo(); @@ -347,7 +350,7 @@ TypeInfo tiledb_dt_info(const ArraySchema &schema, const std::string &name) { /* Helper functions */ /* ****************************** */ -void check_arrow_schema(const ArrowSchema *arw_schema) { +void check_arrow_schema(const ArrowSchema* arw_schema) { if (arw_schema == nullptr) TDB_LERROR("[ArrowIO]: Invalid ArrowSchema object!"); @@ -381,16 +384,22 @@ struct CPPArrowSchema { * Note that an ArrowSchema is *movable*, provided * the release callback of the source is set to null. */ - CPPArrowSchema(std::string name, std::string format, - std::optional metadata, int64_t flags, - std::vector children, - std::shared_ptr dictionary) - : format_(format), name_(name), metadata_(metadata), children_(children), - dictionary_(dictionary) { + CPPArrowSchema( + std::string name, + std::string format, + std::optional metadata, + int64_t flags, + std::vector children, + std::shared_ptr dictionary) + : format_(format) + , name_(name) + , metadata_(metadata) + , children_(children) + , dictionary_(dictionary) { flags_ = flags; n_children_ = children.size(); - schema_ = static_cast(std::malloc(sizeof(ArrowSchema))); + schema_ = static_cast(std::malloc(sizeof(ArrowSchema))); if (schema_ == nullptr) throw tiledb::TileDBError("Failed to allocate ArrowSchema"); @@ -407,17 +416,17 @@ struct CPPArrowSchema { schema_->dictionary = nullptr; // Release callback - schema_->release = ([](ArrowSchema *schema_p) { + schema_->release = ([](ArrowSchema* schema_p) { assert(schema_p->release != nullptr); // Release children for (int64_t i = 0; i < schema_p->n_children; i++) { - ArrowSchema *child_schema = schema_p->children[i]; + ArrowSchema* child_schema = schema_p->children[i]; child_schema->release(child_schema); assert(child_schema->release == nullptr); } // Release dictionary struct - struct ArrowSchema *dict = schema_p->dictionary; + struct ArrowSchema* dict = schema_p->dictionary; if (dict != nullptr && dict->release != nullptr) { dict->release(dict); assert(dict->release == nullptr); @@ -426,14 +435,14 @@ struct CPPArrowSchema { // mark the ArrowSchema struct as released schema_p->release = nullptr; - delete static_cast(schema_p->private_data); + delete static_cast(schema_p->private_data); }); // Private data for release callback schema_->private_data = this; if (n_children_ > 0) { - schema_->children = static_cast(children.data()); + schema_->children = static_cast(children.data()); } if (dictionary) { @@ -460,31 +469,31 @@ struct CPPArrowSchema { * by the ArrowSchema.release callback, which frees the * CPPArrowSchema structure (via ArrowSchema.private_data). */ - void export_ptr(ArrowSchema *out_schema) { + void export_ptr(ArrowSchema* out_schema) { assert(out_schema != nullptr); memcpy(out_schema, schema_, sizeof(ArrowSchema)); std::free(schema_); schema_ = nullptr; } - ArrowSchema *mutable_ptr() { + ArrowSchema* mutable_ptr() { assert(schema_ != nullptr); return schema_; } - ArrowSchema *ptr() const { + ArrowSchema* ptr() const { assert(schema_ != nullptr); return schema_; } -private: - ArrowSchema *schema_; + private: + ArrowSchema* schema_; std::string format_; std::string name_; std::optional metadata_; int64_t flags_; int64_t n_children_; - std::vector children_; + std::vector children_; std::shared_ptr dictionary_; }; @@ -498,12 +507,17 @@ struct CPPArrowArray { * Note that an ArrowSchema is *movable*, provided * the release callback of the source is set to null. */ - CPPArrowArray(int64_t elem_num, int64_t null_num, int64_t offset, - std::vector> children, - std::vector buffers, ArrowAdapter::release_cb cb, - void *cb_data) - : owner_cb(cb), owner_cb_data(cb_data) { - array_ = static_cast(std::malloc(sizeof(ArrowArray))); + CPPArrowArray( + int64_t elem_num, + int64_t null_num, + int64_t offset, + std::vector> children, + std::vector buffers, + ArrowAdapter::release_cb cb, + void* cb_data) + : owner_cb(cb) + , owner_cb_data(cb_data) { + array_ = static_cast(std::malloc(sizeof(ArrowArray))); if (array_ == nullptr) throw tiledb::TileDBError("Failed to allocate ArrowArray"); @@ -517,18 +531,18 @@ struct CPPArrowArray { array_->children = nullptr; array_->dictionary = nullptr; // Bookkeeping - array_->release = ([](ArrowArray *array_p) { + array_->release = ([](ArrowArray* array_p) { assert(array_p->release != nullptr); // Release children for (int64_t i = 0; i < array_p->n_children; i++) { - ArrowArray *child_array = array_p->children[i]; + ArrowArray* child_array = array_p->children[i]; child_array->release(child_array); assert(child_array->release == nullptr); } // Release dictionary - struct ArrowArray *dict = array_p->dictionary; + struct ArrowArray* dict = array_p->dictionary; if (dict != nullptr && dict->release != nullptr) { dict->release(dict); assert(dict->release == nullptr); @@ -538,7 +552,7 @@ struct CPPArrowArray { array_p->release = nullptr; // free the data owner - auto self = static_cast(array_p->private_data); + auto self = static_cast(array_p->private_data); if (self->owner_cb) { self->owner_cb(self->owner_cb_data); } @@ -548,7 +562,7 @@ struct CPPArrowArray { array_->private_data = this; buffers_ = buffers; - array_->buffers = const_cast(buffers_.data()); + array_->buffers = const_cast(buffers_.data()); } /* @@ -564,28 +578,28 @@ struct CPPArrowArray { } } - void export_ptr(ArrowArray *out_array) { + void export_ptr(ArrowArray* out_array) { assert(out_array != nullptr); memcpy(out_array, array_, sizeof(ArrowArray)); std::free(array_); array_ = nullptr; } - ArrowArray *ptr() const { + ArrowArray* ptr() const { assert(array_ != nullptr); return array_; } - ArrowArray *mutable_ptr() { + ArrowArray* mutable_ptr() { assert(array_ != nullptr); return array_; } -private: - ArrowArray *array_; - std::vector buffers_; + private: + ArrowArray* array_; + std::vector buffers_; ArrowAdapter::release_cb owner_cb; - void *owner_cb_data; + void* owner_cb_data; }; /* ****************************** */ @@ -593,19 +607,21 @@ struct CPPArrowArray { /* ****************************** */ class ArrowImporter { -public: - ArrowImporter(Query *const query); + public: + ArrowImporter(Query* const query); ~ArrowImporter(); - void import_(std::string name, ArrowArray *array, ArrowSchema *schema); + void import_(std::string name, ArrowArray* array, ArrowSchema* schema); -private: - Query *const query_; - std::vector offset_buffers_; + private: + Query* const query_; + std::vector offset_buffers_; -}; // class ArrowExporter +}; // class ArrowExporter -ArrowImporter::ArrowImporter(Query *const query) : query_(query) {} +ArrowImporter::ArrowImporter(Query* const query) + : query_(query) { +} ArrowImporter::~ArrowImporter() { for (auto p : offset_buffers_) { @@ -613,8 +629,8 @@ ArrowImporter::~ArrowImporter() { } } -void ArrowImporter::import_(std::string name, ArrowArray *arw_array, - ArrowSchema *arw_schema) { +void ArrowImporter::import_( + std::string name, ArrowArray* arw_array, ArrowSchema* arw_schema) { auto typeinfo = arrow_type_to_tiledb(arw_schema); // buffer conversion @@ -622,31 +638,31 @@ void ArrowImporter::import_(std::string name, ArrowArray *arw_array, if (typeinfo.cell_val_num == TILEDB_VAR_NUM) { assert(arw_array->n_buffers == 3); - void *p_offsets = const_cast(arw_array->buffers[1]); - void *p_data = const_cast(arw_array->buffers[2]); + void* p_offsets = const_cast(arw_array->buffers[1]); + void* p_data = const_cast(arw_array->buffers[2]); const uint64_t num_offsets = arw_array->length; uint64_t data_nbytes = 0; if (typeinfo.arrow_large) { data_nbytes = - static_cast(p_offsets)[num_offsets] * typeinfo.elem_size; + static_cast(p_offsets)[num_offsets] * typeinfo.elem_size; } else { data_nbytes = - static_cast(p_offsets)[num_offsets] * typeinfo.elem_size; + static_cast(p_offsets)[num_offsets] * typeinfo.elem_size; } // Set the TileDB buffer, adding `1` to `num_offsets` to account for // the expected, extra offset. query_->set_data_buffer(name, p_data, data_nbytes); - query_->set_offsets_buffer(name, static_cast(p_offsets), - num_offsets + 1); + query_->set_offsets_buffer( + name, static_cast(p_offsets), num_offsets + 1); } else { // fixed-size attribute (not TILEDB_VAR_NUM) assert(arw_array->n_buffers == 2); - void *p_data = const_cast(arw_array->buffers[1]); + void* p_data = const_cast(arw_array->buffers[1]); uint64_t data_num = arw_array->length; - query_->set_data_buffer(name, static_cast(p_data), data_num); + query_->set_data_buffer(name, static_cast(p_data), data_num); } } @@ -655,27 +671,33 @@ void ArrowImporter::import_(std::string name, ArrowArray *arw_array, /* ****************************** */ class ArrowExporter { -public: - ArrowExporter(Context *const ctx, Query *const query); + public: + ArrowExporter(Context* const ctx, Query* const query); - void export_(const std::string &name, ArrowArray *array, ArrowSchema *schema, - ArrowAdapter::release_cb cb, void *private_data); + void export_( + const std::string& name, + ArrowArray* array, + ArrowSchema* schema, + ArrowAdapter::release_cb cb, + void* private_data); - BufferInfo buffer_info(const std::string &name); + BufferInfo buffer_info(const std::string& name); -private: - Context *const ctx_; - Query *const query_; + private: + Context* const ctx_; + Query* const query_; }; // ArrowExporter implementation -ArrowExporter::ArrowExporter(Context *const ctx, Query *const query) - : ctx_(ctx), query_(query) {} +ArrowExporter::ArrowExporter(Context* const ctx, Query* const query) + : ctx_(ctx) + , query_(query) { +} -BufferInfo ArrowExporter::buffer_info(const std::string &name) { - void *data = nullptr; +BufferInfo ArrowExporter::buffer_info(const std::string& name) { + void* data = nullptr; uint64_t data_nelem = 0; - uint64_t *offsets = nullptr; + uint64_t* offsets = nullptr; uint64_t offsets_nelem = 0; uint64_t elem_size = 0; @@ -703,13 +725,19 @@ BufferInfo ArrowExporter::buffer_info(const std::string &name) { // divide by 4 to get the correct number of offset elements. Note that // the C API does not fetch the data element size, so we ignore // `data_nbytes` below and leave `elem_size` untouched. - uint64_t *offsets_nbytes = nullptr; - uint64_t *data_nbytes = nullptr; - ctx_->handle_error( - tiledb_query_get_data_buffer(ctx_->ptr().get(), query_->ptr().get(), - name.c_str(), &data, &data_nbytes)); + uint64_t* offsets_nbytes = nullptr; + uint64_t* data_nbytes = nullptr; + ctx_->handle_error(tiledb_query_get_data_buffer( + ctx_->ptr().get(), + query_->ptr().get(), + name.c_str(), + &data, + &data_nbytes)); ctx_->handle_error(tiledb_query_get_offsets_buffer( - ctx_->ptr().get(), query_->ptr().get(), name.c_str(), &offsets, + ctx_->ptr().get(), + query_->ptr().get(), + name.c_str(), + &offsets, &offsets_nbytes)); offsets_nelem = *offsets_nbytes / offsets_elem_nbytes; } else { @@ -739,9 +767,12 @@ int64_t flags_for_buffer(BufferInfo binfo) { return 0; } -void ArrowExporter::export_(const std::string &name, ArrowArray *array, - ArrowSchema *schema, ArrowAdapter::release_cb cb, - void *cb_data) { +void ArrowExporter::export_( + const std::string& name, + ArrowArray* array, + ArrowSchema* schema, + ArrowAdapter::release_cb cb, + void* cb_data) { auto bufferinfo = this->buffer_info(name); if (schema == nullptr || array == nullptr) { @@ -755,15 +786,15 @@ void ArrowExporter::export_(const std::string &name, ArrowArray *array, // lifetime: // - address is stored in ArrowSchema.private_data // - delete is called by lambda stored in ArrowSchema.release - CPPArrowSchema *cpp_schema = new CPPArrowSchema( + CPPArrowSchema* cpp_schema = new CPPArrowSchema( name, arrow_fmt.fmt_, std::nullopt, arrow_flags, {}, {}); - std::vector buffers; + std::vector buffers; if (bufferinfo.is_var) { buffers = {nullptr, bufferinfo.offsets, bufferinfo.data}; } else { - cpp_schema = new CPPArrowSchema(name, arrow_fmt.fmt_, std::nullopt, - arrow_flags, {}, {}); + cpp_schema = new CPPArrowSchema( + name, arrow_fmt.fmt_, std::nullopt, arrow_flags, {}, {}); buffers = {nullptr, bufferinfo.data}; } cpp_schema->export_ptr(schema); @@ -783,11 +814,14 @@ void ArrowExporter::export_(const std::string &name, ArrowArray *array, } } - auto cpp_arrow_array = new CPPArrowArray(elem_num, // elem_num - 0, // null_num - 0, // offset - {}, // children - buffers, cb, cb_data); + auto cpp_arrow_array = new CPPArrowArray( + elem_num, // elem_num + 0, // null_num + 0, // offset + {}, // children + buffers, + cb, + cb_data); cpp_arrow_array->export_ptr(array); } @@ -797,8 +831,9 @@ void ArrowExporter::export_(const std::string &name, ArrowArray *array, /* ************************************************************************ */ /* Begin TileDB Arrow IO public API implementation */ -ArrowAdapter::ArrowAdapter(Context *const ctx, Query *const query) - : importer_(nullptr), exporter_(nullptr) { +ArrowAdapter::ArrowAdapter(Context* const ctx, Query* const query) + : importer_(nullptr) + , exporter_(nullptr) { importer_ = new ArrowImporter(query); if (!importer_) { throw tiledb::TileDBError( @@ -811,17 +846,24 @@ ArrowAdapter::ArrowAdapter(Context *const ctx, Query *const query) } } -void ArrowAdapter::export_buffer(const char *name, void *arrow_array, - void *arrow_schema, release_cb cb, - void *private_data) { - exporter_->export_(name, (ArrowArray *)arrow_array, - (ArrowSchema *)arrow_schema, cb, private_data); +void ArrowAdapter::export_buffer( + const char* name, + void* arrow_array, + void* arrow_schema, + release_cb cb, + void* private_data) { + exporter_->export_( + name, + (ArrowArray*)arrow_array, + (ArrowSchema*)arrow_schema, + cb, + private_data); } -void ArrowAdapter::import_buffer(const char *name, void *arrow_array, - void *arrow_schema) { - importer_->import_(name, (ArrowArray *)arrow_array, - (ArrowSchema *)arrow_schema); +void ArrowAdapter::import_buffer( + const char* name, void* arrow_array, void* arrow_schema) { + importer_->import_( + name, (ArrowArray*)arrow_array, (ArrowSchema*)arrow_schema); } ArrowAdapter::~ArrowAdapter() { @@ -831,29 +873,39 @@ ArrowAdapter::~ArrowAdapter() { delete exporter_; } -void query_get_buffer_arrow_array(Context *const ctx, Query *const query, - std::string name, void *v_arw_array, - void *v_arw_schema) { +void query_get_buffer_arrow_array( + Context* const ctx, + Query* const query, + std::string name, + void* v_arw_array, + void* v_arw_schema) { ArrowExporter exporter(ctx, query); - exporter.export_(name, (ArrowArray *)v_arw_array, (ArrowSchema *)v_arw_schema, - nullptr, nullptr); + exporter.export_( + name, + (ArrowArray*)v_arw_array, + (ArrowSchema*)v_arw_schema, + nullptr, + nullptr); } -void query_set_buffer_arrow_array(Query *const query, std::string name, - void *v_arw_array, void *v_arw_schema) { - auto arw_schema = (ArrowSchema *)v_arw_schema; - auto arw_array = (ArrowArray *)v_arw_array; +void query_set_buffer_arrow_array( + Query* const query, + std::string name, + void* v_arw_array, + void* v_arw_schema) { + auto arw_schema = (ArrowSchema*)v_arw_schema; + auto arw_array = (ArrowArray*)v_arw_array; check_arrow_schema(arw_schema); ArrowImporter importer(query); importer.import_(name, arw_array, arw_schema); } -} // end namespace arrow -} // end namespace tiledb +} // end namespace arrow +} // end namespace tiledb /* End TileDB Arrow IO public API implementation */ /* ************************************************************************ */ -#endif // TILEDB_ARROW_H +#endif // TILEDB_ARROW_H diff --git a/tiledb/query_condition.cc b/tiledb/query_condition.cc index 3a640a2f55..81f34884df 100644 --- a/tiledb/query_condition.cc +++ b/tiledb/query_condition.cc @@ -3,8 +3,8 @@ #include +#include // C++ #include "util.h" -#include // C++ #if !defined(NDEBUG) // #include "debug.cc" @@ -18,55 +18,64 @@ namespace py = pybind11; using namespace pybind11::literals; class PyQueryCondition { - -private: + private: Context ctx_; shared_ptr qc_; -public: + public: PyQueryCondition() = delete; PyQueryCondition(py::object ctx) { try { set_ctx(ctx); qc_ = shared_ptr(new QueryCondition(ctx_)); - } catch (TileDBError &e) { + } catch (TileDBError& e) { TPY_ERROR_LOC(e.what()); } } - void init(const string &attribute_name, const string &condition_value, - tiledb_query_condition_op_t op) { + void init( + const string& attribute_name, + const string& condition_value, + tiledb_query_condition_op_t op) { try { qc_->init(attribute_name, condition_value, op); - } catch (TileDBError &e) { + } catch (TileDBError& e) { TPY_ERROR_LOC(e.what()); } } template - void init(const string &attribute_name, T condition_value, - tiledb_query_condition_op_t op) { + void init( + const string& attribute_name, + T condition_value, + tiledb_query_condition_op_t op) { try { qc_->init(attribute_name, &condition_value, sizeof(condition_value), op); - } catch (TileDBError &e) { + } catch (TileDBError& e) { TPY_ERROR_LOC(e.what()); } } - shared_ptr ptr() { return qc_; } + shared_ptr ptr() { + return qc_; + } - py::capsule __capsule__() { return py::capsule(&qc_, "qc"); } + py::capsule __capsule__() { + return py::capsule(&qc_, "qc"); + } void set_use_enumeration(bool use_enumeration) { - QueryConditionExperimental::set_use_enumeration(ctx_, *qc_, - use_enumeration); + QueryConditionExperimental::set_use_enumeration( + ctx_, *qc_, use_enumeration); } template - static PyQueryCondition - create(py::object pyctx, const std::string &field_name, - const std::vector &values, tiledb_query_condition_op_t op) { + static PyQueryCondition create( + py::object pyctx, + const std::string& field_name, + const std::vector& values, + tiledb_query_condition_op_t op) { auto pyqc = PyQueryCondition(pyctx); const Context ctx = std::as_const(pyqc.ctx_); @@ -79,17 +88,20 @@ class PyQueryCondition { return pyqc; } - PyQueryCondition - combine(PyQueryCondition qc, - tiledb_query_condition_combination_op_t combination_op) const { + PyQueryCondition combine( + PyQueryCondition qc, + tiledb_query_condition_combination_op_t combination_op) const { auto pyqc = PyQueryCondition(nullptr, ctx_.ptr().get()); - tiledb_query_condition_t *combined_qc = nullptr; + tiledb_query_condition_t* combined_qc = nullptr; ctx_.handle_error( tiledb_query_condition_alloc(ctx_.ptr().get(), &combined_qc)); ctx_.handle_error(tiledb_query_condition_combine( - ctx_.ptr().get(), qc_->ptr().get(), qc.qc_->ptr().get(), combination_op, + ctx_.ptr().get(), + qc_->ptr().get(), + qc.qc_->ptr().get(), + combination_op, &combined_qc)); pyqc.qc_ = std::shared_ptr( @@ -98,21 +110,21 @@ class PyQueryCondition { return pyqc; } -private: - PyQueryCondition(shared_ptr qc, tiledb_ctx_t *c_ctx) + private: + PyQueryCondition(shared_ptr qc, tiledb_ctx_t* c_ctx) : qc_(qc) { ctx_ = Context(c_ctx, false); } void set_ctx(py::object ctx) { - tiledb_ctx_t *c_ctx; + tiledb_ctx_t* c_ctx; if ((c_ctx = (py::capsule)ctx.attr("__capsule__")()) == nullptr) TPY_ERROR_LOC("Invalid context pointer!") ctx_ = Context(c_ctx, false); } -}; // namespace tiledbpy +}; // namespace tiledbpy -void init_query_condition(py::module &m) { +void init_query_condition(py::module& m) { py::class_(m, "PyQueryCondition") .def(py::init(), py::arg("ctx") = py::none()) @@ -121,50 +133,61 @@ void init_query_condition(py::module &m) { * https://github.com/pybind/pybind11/issues/1667 */ - .def("init_string", - static_cast( - &PyQueryCondition::init)) - .def("init_uint64", - static_cast( - &PyQueryCondition::init)) - .def("init_int64", - static_cast( - &PyQueryCondition::init)) - .def("init_uint32", - static_cast( - &PyQueryCondition::init)) - .def("init_int32", - static_cast( - &PyQueryCondition::init)) - .def("init_uint16", - static_cast( - &PyQueryCondition::init)) - .def("init_int16", - static_cast( - &PyQueryCondition::init)) - .def("init_uint8", - static_cast( - &PyQueryCondition::init)) - .def("init_int8", - static_cast( - &PyQueryCondition::init)) - .def("init_float32", - static_cast( - &PyQueryCondition::init)) - .def("init_float64", - static_cast( - &PyQueryCondition::init)) + .def( + "init_string", + static_cast( + &PyQueryCondition::init)) + .def( + "init_uint64", + static_cast( + &PyQueryCondition::init)) + .def( + "init_int64", + static_cast( + &PyQueryCondition::init)) + .def( + "init_uint32", + static_cast( + &PyQueryCondition::init)) + .def( + "init_int32", + static_cast( + &PyQueryCondition::init)) + .def( + "init_uint16", + static_cast( + &PyQueryCondition::init)) + .def( + "init_int16", + static_cast( + &PyQueryCondition::init)) + .def( + "init_uint8", + static_cast( + &PyQueryCondition::init)) + .def( + "init_int8", + static_cast( + &PyQueryCondition::init)) + .def( + "init_float32", + static_cast( + &PyQueryCondition::init)) + .def( + "init_float64", + static_cast( + &PyQueryCondition::init)) .def("__capsule__", &PyQueryCondition::__capsule__) @@ -173,61 +196,83 @@ void init_query_condition(py::module &m) { .def_static( "create_string", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_uint64", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_int64", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_uint32", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_int32", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_uint16", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_int8", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_uint16", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_int8", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_float32", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)) .def_static( "create_float64", static_cast &, + py::object, + const std::string&, + const std::vector&, tiledb_query_condition_op_t)>(&PyQueryCondition::create)); - py::enum_(m, "tiledb_query_condition_op_t", - py::arithmetic()) + py::enum_( + m, "tiledb_query_condition_op_t", py::arithmetic()) .value("TILEDB_LT", TILEDB_LT) .value("TILEDB_LE", TILEDB_LE) .value("TILEDB_GT", TILEDB_GT) @@ -244,4 +289,4 @@ void init_query_condition(py::module &m) { .value("TILEDB_OR", TILEDB_OR) .export_values(); } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/schema_evolution.cc b/tiledb/schema_evolution.cc index 89889e0adc..f5a7d7a247 100644 --- a/tiledb/schema_evolution.cc +++ b/tiledb/schema_evolution.cc @@ -10,20 +10,20 @@ namespace tiledbpy { namespace py = pybind11; typedef struct { - tiledb_ctx_t *ctx_; - tiledb_array_schema_evolution_t *evol_; + tiledb_ctx_t* ctx_; + tiledb_array_schema_evolution_t* evol_; } PyArraySchemaEvolution; using ArraySchemaEvolution = PyArraySchemaEvolution; -void init_schema_evolution(py::module &m) { +void init_schema_evolution(py::module& m) { py::class_(m, "ArraySchemaEvolution") .def(py::init([](py::object ctx_py) { - tiledb_ctx_t *ctx_c = (py::capsule)ctx_py.attr("__capsule__")(); + tiledb_ctx_t* ctx_c = (py::capsule)ctx_py.attr("__capsule__")(); if (ctx_c == nullptr) TPY_ERROR_LOC("Invalid context pointer"); - tiledb_array_schema_evolution_t *evol_p; + tiledb_array_schema_evolution_t* evol_p; int rc = tiledb_array_schema_evolution_alloc(ctx_c, &evol_p); if (rc != TILEDB_OK) { TPY_ERROR_LOC(get_last_ctx_err_str(ctx_c, rc)); @@ -31,90 +31,98 @@ void init_schema_evolution(py::module &m) { return new PyArraySchemaEvolution({ctx_c, evol_p}); })) - .def("add_attribute", - [](ArraySchemaEvolution &inst, py::object attr_py) { - tiledb_attribute_t *attr_c = - (py::capsule)attr_py.attr("__capsule__")(); - if (attr_c == nullptr) - TPY_ERROR_LOC("Invalid Attribute!"); + .def( + "add_attribute", + [](ArraySchemaEvolution& inst, py::object attr_py) { + tiledb_attribute_t* attr_c = + (py::capsule)attr_py.attr("__capsule__")(); + if (attr_c == nullptr) + TPY_ERROR_LOC("Invalid Attribute!"); - int rc = tiledb_array_schema_evolution_add_attribute( - inst.ctx_, inst.evol_, attr_c); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("drop_attribute", - [](ArraySchemaEvolution &inst, std::string attr_name) { - int rc = tiledb_array_schema_evolution_drop_attribute( - inst.ctx_, inst.evol_, attr_name.c_str()); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("array_evolve", - [](ArraySchemaEvolution &inst, std::string uri) { - int rc = tiledb_array_evolve(inst.ctx_, uri.c_str(), inst.evol_); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("set_timestamp_range", - [](ArraySchemaEvolution &inst, uint64_t timestamp) { - int rc = tiledb_array_schema_evolution_set_timestamp_range( - inst.ctx_, inst.evol_, timestamp, timestamp); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("add_enumeration", - [](ArraySchemaEvolution &inst, py::object enum_py) { - tiledb_enumeration_t *enum_c = - (py::capsule)enum_py.attr("__capsule__")(); - if (enum_c == nullptr) - TPY_ERROR_LOC("Invalid Enumeration!"); - int rc = tiledb_array_schema_evolution_add_enumeration( - inst.ctx_, inst.evol_, enum_c); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("drop_enumeration", - [](ArraySchemaEvolution &inst, const std::string &enumeration_name) { - int rc = tiledb_array_schema_evolution_drop_enumeration( - inst.ctx_, inst.evol_, enumeration_name.c_str()); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) - .def("extend_enumeration", - [](ArraySchemaEvolution &inst, py::object enum_py) { - tiledb_enumeration_t *enum_c = - (py::capsule)enum_py.attr("__capsule__")(); - if (enum_c == nullptr) - TPY_ERROR_LOC("Invalid Enumeration!"); - int rc = tiledb_array_schema_evolution_extend_enumeration( - inst.ctx_, inst.evol_, enum_c); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) + int rc = tiledb_array_schema_evolution_add_attribute( + inst.ctx_, inst.evol_, attr_c); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "drop_attribute", + [](ArraySchemaEvolution& inst, std::string attr_name) { + int rc = tiledb_array_schema_evolution_drop_attribute( + inst.ctx_, inst.evol_, attr_name.c_str()); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "array_evolve", + [](ArraySchemaEvolution& inst, std::string uri) { + int rc = tiledb_array_evolve(inst.ctx_, uri.c_str(), inst.evol_); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "set_timestamp_range", + [](ArraySchemaEvolution& inst, uint64_t timestamp) { + int rc = tiledb_array_schema_evolution_set_timestamp_range( + inst.ctx_, inst.evol_, timestamp, timestamp); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "add_enumeration", + [](ArraySchemaEvolution& inst, py::object enum_py) { + tiledb_enumeration_t* enum_c = + (py::capsule)enum_py.attr("__capsule__")(); + if (enum_c == nullptr) + TPY_ERROR_LOC("Invalid Enumeration!"); + int rc = tiledb_array_schema_evolution_add_enumeration( + inst.ctx_, inst.evol_, enum_c); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "drop_enumeration", + [](ArraySchemaEvolution& inst, const std::string& enumeration_name) { + int rc = tiledb_array_schema_evolution_drop_enumeration( + inst.ctx_, inst.evol_, enumeration_name.c_str()); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) + .def( + "extend_enumeration", + [](ArraySchemaEvolution& inst, py::object enum_py) { + tiledb_enumeration_t* enum_c = + (py::capsule)enum_py.attr("__capsule__")(); + if (enum_c == nullptr) + TPY_ERROR_LOC("Invalid Enumeration!"); + int rc = tiledb_array_schema_evolution_extend_enumeration( + inst.ctx_, inst.evol_, enum_c); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 25 - .def("expand_current_domain", - [](ArraySchemaEvolution &inst, py::object current_domain_py) { - tiledb_current_domain_t *current_domain_c = - (py::capsule)current_domain_py.attr("__capsule__")(); - if (current_domain_c == nullptr) - TPY_ERROR_LOC("Invalid Current Domain!"); - int rc = tiledb_array_schema_evolution_expand_current_domain( - inst.ctx_, inst.evol_, current_domain_c); - if (rc != TILEDB_OK) { - TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); - } - }) + .def( + "expand_current_domain", + [](ArraySchemaEvolution& inst, py::object current_domain_py) { + tiledb_current_domain_t* current_domain_c = + (py::capsule)current_domain_py.attr("__capsule__")(); + if (current_domain_c == nullptr) + TPY_ERROR_LOC("Invalid Current Domain!"); + int rc = tiledb_array_schema_evolution_expand_current_domain( + inst.ctx_, inst.evol_, current_domain_c); + if (rc != TILEDB_OK) { + TPY_ERROR_LOC(get_last_ctx_err_str(inst.ctx_, rc)); + } + }) #endif ; } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/serialization.cc b/tiledb/serialization.cc index 709dfc7288..e0109714b9 100644 --- a/tiledb/serialization.cc +++ b/tiledb/serialization.cc @@ -6,9 +6,9 @@ #include +#include // C +#include // C++ #include "util.h" -#include // C++ -#include // C #if !defined(NDEBUG) // #include "debug.cc" @@ -22,18 +22,19 @@ namespace py = pybind11; using namespace pybind11::literals; class PySerialization { - -public: - static void *deserialize_query(py::object ctx, py::object array, - py::buffer buffer, - tiledb_serialization_type_t serialize_type, - int32_t client_side) { + public: + static void* deserialize_query( + py::object ctx, + py::object array, + py::buffer buffer, + tiledb_serialization_type_t serialize_type, + int32_t client_side) { int rc; - tiledb_ctx_t *ctx_c; - tiledb_array_t *arr_c; - tiledb_query_t *qry_c; - tiledb_buffer_t *buf_c; + tiledb_ctx_t* ctx_c; + tiledb_array_t* arr_c; + tiledb_query_t* qry_c; + tiledb_buffer_t* buf_c; ctx_c = (py::capsule)ctx.attr("__capsule__")(); if (ctx_c == nullptr) @@ -56,8 +57,8 @@ class PySerialization { if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not set buffer."); - rc = tiledb_deserialize_query(ctx_c, buf_c, serialize_type, client_side, - qry_c); + rc = tiledb_deserialize_query( + ctx_c, buf_c, serialize_type, client_side, qry_c); if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not deserialize query."); @@ -65,15 +66,15 @@ class PySerialization { } }; -void init_serialization(py::module &m) { +void init_serialization(py::module& m) { py::class_(m, "serialization") .def_static("deserialize_query", &PySerialization::deserialize_query); - py::enum_(m, "tiledb_serialization_type_t", - py::arithmetic()) + py::enum_( + m, "tiledb_serialization_type_t", py::arithmetic()) .value("TILEDB_CAPNP", TILEDB_CAPNP) .value("TILEDB_JSON", TILEDB_JSON) .export_values(); } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/tests/test_metadata.cc b/tiledb/tests/test_metadata.cc index f93c3d2d96..b4d755e021 100644 --- a/tiledb/tests/test_metadata.cc +++ b/tiledb/tests/test_metadata.cc @@ -9,8 +9,8 @@ #define TILEDB_DEPRECATED #define TILEDB_DEPRECATED_EXPORT +#include // C++ #include "../util.h" -#include // C++ #if !defined(NDEBUG) // #include "debug.cc" @@ -24,8 +24,7 @@ namespace py = pybind11; using namespace pybind11::literals; class PyASCIIMetadataTest { - -public: + public: static void write_ascii(py::str uri) { Context ctx; Array array(ctx, uri, TILEDB_WRITE); @@ -37,9 +36,9 @@ class PyASCIIMetadataTest { } }; -void init_test_metadata(py::module &m) { +void init_test_metadata(py::module& m) { py::class_(m, "metadata_test_aux") .def_static("write_ascii", &PyASCIIMetadataTest::write_ascii); } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/tests/test_serialization.cc b/tiledb/tests/test_serialization.cc index 31df7663a0..d167996c8f 100644 --- a/tiledb/tests/test_serialization.cc +++ b/tiledb/tests/test_serialization.cc @@ -9,9 +9,9 @@ #define TILEDB_DEPRECATED #define TILEDB_DEPRECATED_EXPORT +#include // C +#include // C++ #include "../util.h" -#include // C++ -#include // C #if !defined(NDEBUG) // #include "debug.cc" @@ -25,14 +25,13 @@ namespace py = pybind11; using namespace pybind11::literals; class PySerializationTest { - -public: - static py::bytes create_serialized_test_query(py::object pyctx, - py::object pyarray) { + public: + static py::bytes create_serialized_test_query( + py::object pyctx, py::object pyarray) { int rc; - tiledb_ctx_t *ctx; - tiledb_array_t *array; + tiledb_ctx_t* ctx; + tiledb_array_t* array; ctx = (py::capsule)pyctx.attr("__capsule__")(); if (ctx == nullptr) @@ -47,18 +46,18 @@ class PySerializationTest { int64_t data[5]; uint64_t data_size = sizeof(data); - tiledb_subarray_t *subarray; + tiledb_subarray_t* subarray; tiledb_subarray_alloc(ctx, array, &subarray); tiledb_subarray_set_subarray(ctx, subarray, &subarray_v); - tiledb_query_t *query; + tiledb_query_t* query; tiledb_query_alloc(ctx, array, TILEDB_READ, &query); tiledb_query_set_subarray_t(ctx, query, subarray); tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); tiledb_query_set_data_buffer(ctx, query, "", data, &data_size); - tiledb_buffer_list_t *buff_list; - tiledb_buffer_t *buff; + tiledb_buffer_list_t* buff_list; + tiledb_buffer_t* buff; rc = tiledb_serialize_query(ctx, query, TILEDB_CAPNP, 1, &buff_list); if (rc == TILEDB_ERR) @@ -68,14 +67,14 @@ class PySerializationTest { if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not flatten the buffer list."); - void *buff_data; + void* buff_data; uint64_t buff_num_bytes; rc = tiledb_buffer_get_data(ctx, buff, &buff_data, &buff_num_bytes); if (rc == TILEDB_ERR) TPY_ERROR_LOC("Could not get the data from the buffer."); - py::bytes output((char *)buff_data, buff_num_bytes); + py::bytes output((char*)buff_data, buff_num_bytes); tiledb_buffer_free(&buff); tiledb_buffer_list_free(&buff_list); @@ -86,10 +85,11 @@ class PySerializationTest { } }; -void init_test_serialization(py::module &m) { +void init_test_serialization(py::module& m) { py::class_(m, "test_serialization") - .def_static("create_serialized_test_query", - &PySerializationTest::create_serialized_test_query); + .def_static( + "create_serialized_test_query", + &PySerializationTest::create_serialized_test_query); } -}; // namespace tiledbpy +}; // namespace tiledbpy diff --git a/tiledb/tests/test_webp.cc b/tiledb/tests/test_webp.cc index 68c0ef43b1..cbb5559af1 100644 --- a/tiledb/tests/test_webp.cc +++ b/tiledb/tests/test_webp.cc @@ -8,12 +8,12 @@ using namespace tiledb; namespace py = pybind11; class WebpFilter { -public: + public: static bool webp_filter_exists() { Context ctx; try { auto f = Filter(ctx, TILEDB_FILTER_WEBP); - } catch (TileDBError &) { + } catch (TileDBError&) { // Can't create WebP filter; built with TILEDB_WEBP=OFF return false; } @@ -21,9 +21,9 @@ class WebpFilter { } }; -void init_test_webp_filter(py::module &m) { +void init_test_webp_filter(py::module& m) { py::class_(m, "test_webp_filter") .def_static("webp_filter_exists", &WebpFilter::webp_filter_exists); } -}; // namespace tiledbpy \ No newline at end of file +}; // namespace tiledbpy diff --git a/tiledb/util.cc b/tiledb/util.cc index afa00048cf..a0ce6d7a5e 100644 --- a/tiledb/util.cc +++ b/tiledb/util.cc @@ -2,8 +2,8 @@ #include -std::string _get_tiledb_err_str(tiledb_error_t *err_ptr) { - const char *err_msg_ptr = NULL; +std::string _get_tiledb_err_str(tiledb_error_t* err_ptr) { + const char* err_msg_ptr = NULL; int ret = tiledb_error_message(err_ptr, &err_msg_ptr); if (ret != TILEDB_OK) { @@ -16,11 +16,11 @@ std::string _get_tiledb_err_str(tiledb_error_t *err_ptr) { return std::string(err_msg_ptr); } -std::string get_last_ctx_err_str(tiledb_ctx_t *ctx_ptr, int rc) { +std::string get_last_ctx_err_str(tiledb_ctx_t* ctx_ptr, int rc) { if (rc == TILEDB_OOM) throw std::bad_alloc(); - tiledb_error_t *err_ptr = NULL; + tiledb_error_t* err_ptr = NULL; int ret = tiledb_ctx_get_last_error(ctx_ptr, &err_ptr); if (ret != TILEDB_OK) { @@ -31,4 +31,4 @@ std::string get_last_ctx_err_str(tiledb_ctx_t *ctx_ptr, int rc) { return "error retrieving error object from ctx"; } return _get_tiledb_err_str(err_ptr); -} \ No newline at end of file +} diff --git a/tiledb/util.h b/tiledb/util.h index cb234fbe04..e4914b2dee 100644 --- a/tiledb/util.h +++ b/tiledb/util.h @@ -7,27 +7,32 @@ const uint64_t DEFAULT_INIT_BUFFER_BYTES = 1310720 * 8; const uint64_t DEFAULT_ALLOC_MAX_BYTES = uint64_t(5 * pow(2, 30)); -std::string get_last_ctx_err_str(tiledb_ctx_t *, int); +std::string get_last_ctx_err_str(tiledb_ctx_t*, int); -#define TPY_ERROR_STR(m) \ +#define TPY_ERROR_STR(m) \ [](auto m) -> std::string { \ return std::string(m) + " (" + __FILE__ + ":" + \ - std::to_string(__LINE__) + ")"); \ + std::to_string(__LINE__) + ")"); \ }(); -#define TPY_ERROR_LOC(m) \ - throw TileDBPyError(std::string(m) + " (" + __FILE__ + ":" + \ - std::to_string(__LINE__) + ")"); +#define TPY_ERROR_LOC(m) \ + throw TileDBPyError( \ + std::string(m) + " (" + __FILE__ + ":" + std::to_string(__LINE__) + \ + ")"); class TileDBPyError : std::runtime_error { -public: - explicit TileDBPyError(const char *m) : std::runtime_error(m) {} - explicit TileDBPyError(std::string m) : std::runtime_error(m.c_str()) {} + public: + explicit TileDBPyError(const char* m) + : std::runtime_error(m) { + } + explicit TileDBPyError(std::string m) + : std::runtime_error(m.c_str()) { + } -public: - virtual const char *what() const noexcept override { + public: + virtual const char* what() const noexcept override { return std::runtime_error::what(); } }; -#endif // TILEDB_PY_UTIL_H +#endif // TILEDB_PY_UTIL_H