Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update C++ dependencies, roughly matching ones used in python. #5472

Merged
merged 14 commits into from
May 28, 2024
Merged

Update C++ dependencies, roughly matching ones used in python. #5472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
db3150c
Update C++ dependencies, roughly matching ones used in python.
jcferretti May 10, 2024
9f85447
Update protoc-base.
jcferretti May 10, 2024
f86969b
Recover comment lost by mistake.
jcferretti May 10, 2024
26a497c
Updated protoc version.
jcferretti May 17, 2024
bf73245
Update protobuf.
jcferretti May 17, 2024
404fbff
getting closer...
jcferretti May 17, 2024
519f590
Update protoc base image.
jcferretti May 23, 2024
292b676
Update Classpaths.groovy.
jcferretti May 23, 2024
419fde6
One more place.
jcferretti May 23, 2024
d65cc51
One more more place.
jcferretti May 23, 2024
ae48d49
Update C++ protos.
jcferretti May 23, 2024
b1f7138
Update cpp-clients-multi base image.
jcferretti May 24, 2024
13f60bb
Update build-dependencies.sh.
jcferretti May 24, 2024
8c031e3
Move R client to new flight API.
jcferretti May 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions R/rdeephaven/src/client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,10 +540,9 @@ class TableHandleWrapper {

std::shared_ptr<arrow::flight::FlightStreamReader> fsr = internal_tbl_hdl.GetFlightStreamReader();

std::vector<std::shared_ptr<arrow::RecordBatch>> empty_record_batches;
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(fsr->ReadAll(&empty_record_batches)));
arrow::Result<arrow::RecordBatchVector> record_batches = fsr->ToRecordBatches();

std::shared_ptr<arrow::RecordBatchReader> record_batch_reader = arrow::RecordBatchReader::Make(empty_record_batches).ValueOrDie();
std::shared_ptr<arrow::RecordBatchReader> record_batch_reader = arrow::RecordBatchReader::Make(std::move(*record_batches)).ValueOrDie();
ArrowArrayStream* stream_ptr = new ArrowArrayStream();
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(arrow::ExportRecordBatchReader(record_batch_reader, stream_ptr)));

Expand Down Expand Up @@ -705,23 +704,21 @@ class ClientWrapper {
auto schema = record_batch_reader.get()->schema();

// write RecordBatchReader data to table on server with DoPut
std::unique_ptr<arrow::flight::FlightStreamWriter> fsw;
std::unique_ptr<arrow::flight::FlightMetadataReader> fmr;

auto ticket = internal_tbl_hdl_mngr.NewTicket();
auto fd = deephaven::client::utility::ArrowUtil::ConvertTicketToFlightDescriptor(ticket);
arrow::Result<arrow::flight::FlightClient::DoPutResult> r = wrapper.FlightClient()->DoPut(options, fd, schema);

deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(wrapper.FlightClient()->DoPut(options, fd, schema, &fsw, &fmr)));
while(true) {
std::shared_ptr<arrow::RecordBatch> this_batch;
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(record_batch_reader->ReadNext(&this_batch)));
if (this_batch == nullptr) {
break;
}
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(fsw->WriteRecordBatch(*this_batch)));
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(r->writer->WriteRecordBatch(*this_batch)));
}
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(fsw->DoneWriting()));
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(fsw->Close()));
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(r->writer->DoneWriting()));
deephaven::client::utility::OkOrThrow(DEEPHAVEN_LOCATION_EXPR(r->writer->Close()));

auto new_tbl_hdl = internal_tbl_hdl_mngr.MakeTableHandleFromTicket(ticket);
return new TableHandleWrapper(new_tbl_hdl);
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/groovy/Classpaths.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class Classpaths {
// TODO(deephaven-core#1685): Create strategy around updating and maintaining protoc version
static final String PROTOBUF_GROUP = 'com.google.protobuf'
static final String PROTOBUF_NAME = 'protobuf-java'
static final String PROTOBUF_VERSION = '3.23.0'
static final String PROTOBUF_VERSION = '3.25.3'

// See dependency matrix for particular gRPC versions at https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty
static final String BORINGSSL_GROUP = 'io.netty'
Expand Down
28 changes: 15 additions & 13 deletions cpp-client/build-dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,8 +490,8 @@ if [ "$CLONE_ABSEIL" = "yes" ]; then
echo
echo "*** Clone abseil"
cd $SRC
# Previously used version: 20210324.2
git clone $GIT_FLAGS -b 20211102.0 --depth 1 "${GITHUB_BASE_URL}/abseil/abseil-cpp.git"
# Previously used version: 20211102.0
git clone $GIT_FLAGS -b 20240116.0 --depth 1 "${GITHUB_BASE_URL}/abseil/abseil-cpp.git"
echo "*** Cloning abseil DONE"
if [ "$fedora38" = "yes" ]; then
echo "*** Patching abseil for Fedora 38"
Expand All @@ -513,6 +513,7 @@ if [ "$BUILD_ABSEIL" = "yes" ]; then
cd $SRC/abseil-cpp
mkdir -p "cmake/$BUILD_DIR" && cd "cmake/$BUILD_DIR"
cmake -DCMAKE_CXX_STANDARD=17 \
-DABSL_PROPAGATE_CXX_STD=ON \
${cmake_common_args} \
-DCMAKE_INSTALL_PREFIX=$(prefix abseil) \
../..
Expand All @@ -530,8 +531,8 @@ if [ "$CLONE_ZLIB" = "yes" ]; then
echo
echo "*** Clone zlib"
cd $SRC
# Previously used version: v1.2.11
git clone $GIT_FLAGS -b v1.2.13 --depth 1 "${GITHUB_BASE_URL}/madler/zlib"
# Previously used version: v1.2.13
git clone $GIT_FLAGS -b v1.3 --depth 1 "${GITHUB_BASE_URL}/madler/zlib"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to specify the patch version, or are we ok with that changing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a specific version, is just the name they gave, in git terms is just some arbitrary string. Mean as far as I understand how the universe works, that won't change when they create a new version.

echo "*** Cloning zlib DONE"
fi
if [ "$BUILD_ZLIB" = "yes" ]; then
Expand Down Expand Up @@ -561,24 +562,25 @@ if [ "$CLONE_PROTOBUF" = "yes" ]; then
echo
echo "*** Cloning protobuf"
cd $SRC
# Previously used version: v3.21.12
git clone $GIT_FLAGS -b v3.21.2 --depth 1 "${GITHUB_BASE_URL}/protocolbuffers/protobuf.git"
# Previously used version: v3.21.2
git clone $GIT_FLAGS -b v25.3 --depth 1 "${GITHUB_BASE_URL}/protocolbuffers/protobuf.git"
Comment on lines +565 to +566
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Were we ultra out of date?
  2. Did they change their versioning convention?
  3. Do we need to specify the patch version?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. No, they changed in weird ways they numbering, and managed to confuse a lot of people in the process. They also introduced some backwards incompatibilities which is why we can't go directly to 26
  2. Yes, see above.
  3. No, like the previous answer, that tag should not change.

echo "*** Cloning protobuf DONE"
fi
if [ "$BUILD_PROTOBUF" = "yes" ]; then
echo
echo "*** Building protobuf"
cd $SRC/protobuf
# Note in the previously used version we built inside cmake/build; not anymore.
mkdir -p "cmake/$BUILD_DIR" && cd "cmake/$BUILD_DIR"
mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
cmake -Dprotobuf_BUILD_TESTS=OFF \
${cmake_common_args} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-Dprotobuf_ABSL_PROVIDER=package \
-DCMAKE_INSTALL_PREFIX=$(prefix protobuf) \
..
make -j$NCPUS
make install
cd ../.. && rm -fr "cmake/$BUILD_DIR"
cd .. && rm -fr "$BUILD_DIR"
if [ "$clean" = "yes" ]; then
rm -fr "$SRC/protobuf"
fi
Expand Down Expand Up @@ -689,7 +691,7 @@ if [ "$CLONE_CARES" = "yes" ]; then
echo "*** Clone ares"
cd $SRC
# Previously used version: cares-1_18_1
git clone $GIT_FLAGS -b cares-1_18_1 --depth 1 "${GITHUB_BASE_URL}/c-ares/c-ares.git"
git clone $GIT_FLAGS -b cares-1_28_1 --depth 1 "${GITHUB_BASE_URL}/c-ares/c-ares.git"
echo "*** Cloning ares DONE"
fi
if [ "$BUILD_CARES" = "yes" ]; then
Expand Down Expand Up @@ -721,8 +723,8 @@ if [ "$CLONE_GRPC" = "yes" ]; then
echo
echo "*** Clone grpc"
cd $SRC
# Previously used version: v1.45.2
git clone $GIT_FLAGS -b v1.46.7 --depth 1 "${GITHUB_BASE_URL}/grpc/grpc"
# Previously used version: v1.46.7
git clone $GIT_FLAGS -b v1.63.0 --depth 1 "${GITHUB_BASE_URL}/grpc/grpc"
echo "*** Cloning grpc DONE"
fi
if [ "$BUILD_GRPC" = "yes" ]; then
Expand Down Expand Up @@ -756,8 +758,8 @@ if [ "$CLONE_ARROW" = "yes" ]; then
echo
echo "*** Cloning arrow"
cd $SRC
# Previously used version: apache-arrow-7.0.0
git clone $GIT_FLAGS -b apache-arrow-13.0.0 --depth 1 "${GITHUB_BASE_URL}/apache/arrow"
# Previously used version: apache-arrow-13.0.0
git clone $GIT_FLAGS -b apache-arrow-16.0.0 --depth 1 "${GITHUB_BASE_URL}/apache/arrow"
echo "*** Cloning arrow DONE"
fi
if [ "$BUILD_ARROW" = "yes" ]; then
Expand Down
26 changes: 13 additions & 13 deletions cpp-client/deephaven/dhclient/proto/deephaven/proto/application.grpc.pb.cc
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 17 additions & 17 deletions cpp-client/deephaven/dhclient/proto/deephaven/proto/application.grpc.pb.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading