From d86d679ae9569f373182522cbe3bf3364b448754 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Mon, 22 Jun 2020 16:53:59 -0400 Subject: [PATCH 1/4] cli/flags,config: new flag for tenant KV listen addr This patch introduces the ability to split off the tenant KV server into a separate port, using the new command-line flag `--tenant-addr`. This is motivated primarily by security concerns. We plan to use different TLS configs for KV<->KV kv/replication/gossip traffic and SQL<->KV tenant kv traffic. The easiest way to facilitate this is to listen to tenant KV traffic on a separate port and separate gRPC server. The `--tenant-addr` and `--advertise-tenant-addr` flags are inspired by the `--sql-addr` and `--advertise-sql-addr` flags. The two pairs have similar fallback behavior, as is discussed in depth in 6cb71bf. --- c-deps/libroach/protos/roachpb/metadata.pb.cc | 78 ++++- c-deps/libroach/protos/roachpb/metadata.pb.h | 79 +++++- pkg/base/addr_validation.go | 16 ++ pkg/base/addr_validation_test.go | 120 ++++---- pkg/base/config.go | 22 +- pkg/base/test_server_args.go | 2 + pkg/cli/cliflags/flags.go | 101 +++++-- pkg/cli/flags.go | 43 +++ pkg/cli/flags_test.go | 110 ++++++- pkg/cli/zip.go | 7 +- pkg/roachpb/metadata.go | 23 ++ pkg/roachpb/metadata.pb.go | 268 ++++++++++-------- pkg/roachpb/metadata.proto | 6 +- pkg/rpc/tls.go | 3 + pkg/server/drain.go | 6 + pkg/server/node.go | 3 +- pkg/server/server.go | 124 ++++++-- pkg/server/server_test.go | 12 +- pkg/server/status/recorder.go | 4 +- pkg/server/status/recorder_test.go | 2 +- pkg/server/testserver.go | 39 ++- pkg/sql/crdb_internal.go | 6 +- pkg/testutils/serverutils/test_server_shim.go | 7 + 23 files changed, 808 insertions(+), 273 deletions(-) diff --git a/c-deps/libroach/protos/roachpb/metadata.pb.cc b/c-deps/libroach/protos/roachpb/metadata.pb.cc index 0dd2b4ff83ce..0b89ce5785a9 100644 --- a/c-deps/libroach/protos/roachpb/metadata.pb.cc +++ b/c-deps/libroach/protos/roachpb/metadata.pb.cc @@ -2507,6 +2507,8 @@ void NodeDescriptor::InitAsDefaultInstance() { ::cockroach::roachpb::Version::internal_default_instance()); ::cockroach::roachpb::_NodeDescriptor_default_instance_._instance.get_mutable()->sql_address_ = const_cast< ::cockroach::util::UnresolvedAddr*>( ::cockroach::util::UnresolvedAddr::internal_default_instance()); + ::cockroach::roachpb::_NodeDescriptor_default_instance_._instance.get_mutable()->tenant_address_ = const_cast< ::cockroach::util::UnresolvedAddr*>( + ::cockroach::util::UnresolvedAddr::internal_default_instance()); } void NodeDescriptor::clear_address() { if (address_ != NULL) address_->Clear(); @@ -2516,6 +2518,10 @@ void NodeDescriptor::clear_sql_address() { if (sql_address_ != NULL) sql_address_->Clear(); clear_has_sql_address(); } +void NodeDescriptor::clear_tenant_address() { + if (tenant_address_ != NULL) tenant_address_->Clear(); + clear_has_tenant_address(); +} #if !defined(_MSC_VER) || _MSC_VER >= 1900 const int NodeDescriptor::kNodeIdFieldNumber; const int NodeDescriptor::kAddressFieldNumber; @@ -2527,6 +2533,7 @@ const int NodeDescriptor::kStartedAtFieldNumber; const int NodeDescriptor::kLocalityAddressFieldNumber; const int NodeDescriptor::kClusterNameFieldNumber; const int NodeDescriptor::kSqlAddressFieldNumber; +const int NodeDescriptor::kTenantAddressFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 NodeDescriptor::NodeDescriptor() @@ -2575,6 +2582,11 @@ NodeDescriptor::NodeDescriptor(const NodeDescriptor& from) } else { sql_address_ = NULL; } + if (from.has_tenant_address()) { + tenant_address_ = new ::cockroach::util::UnresolvedAddr(*from.tenant_address_); + } else { + tenant_address_ = NULL; + } ::memcpy(&started_at_, &from.started_at_, static_cast(reinterpret_cast(&node_id_) - reinterpret_cast(&started_at_)) + sizeof(node_id_)); @@ -2602,6 +2614,7 @@ void NodeDescriptor::SharedDtor() { if (this != internal_default_instance()) delete locality_; if (this != internal_default_instance()) delete serverversion_; if (this != internal_default_instance()) delete sql_address_; + if (this != internal_default_instance()) delete tenant_address_; } void NodeDescriptor::SetCachedSize(int size) const { @@ -2621,7 +2634,7 @@ void NodeDescriptor::Clear() { locality_address_.Clear(); cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 127u) { + if (cached_has_bits & 255u) { if (cached_has_bits & 0x00000001u) { build_tag_.ClearNonDefaultToEmptyNoArena(); } @@ -2648,9 +2661,16 @@ void NodeDescriptor::Clear() { GOOGLE_DCHECK(sql_address_ != NULL); sql_address_->Clear(); } + if (cached_has_bits & 0x00000080u) { + GOOGLE_DCHECK(tenant_address_ != NULL); + tenant_address_->Clear(); + } + } + if (cached_has_bits & 768u) { + ::memset(&started_at_, 0, static_cast( + reinterpret_cast(&node_id_) - + reinterpret_cast(&started_at_)) + sizeof(node_id_)); } - started_at_ = GOOGLE_LONGLONG(0); - node_id_ = 0; _has_bits_.Clear(); _internal_metadata_.Clear(); } @@ -2785,6 +2805,17 @@ bool NodeDescriptor::MergePartialFromCodedStream( break; } + case 11: { + if (static_cast< ::google::protobuf::uint8>(tag) == + static_cast< ::google::protobuf::uint8>(90u /* 90 & 0xFF */)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( + input, mutable_tenant_address())); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -2812,7 +2843,7 @@ void NodeDescriptor::SerializeWithCachedSizes( (void) cached_has_bits; cached_has_bits = _has_bits_[0]; - if (cached_has_bits & 0x00000100u) { + if (cached_has_bits & 0x00000200u) { ::google::protobuf::internal::WireFormatLite::WriteInt32(1, this->node_id(), output); } @@ -2841,7 +2872,7 @@ void NodeDescriptor::SerializeWithCachedSizes( 6, this->build_tag(), output); } - if (cached_has_bits & 0x00000080u) { + if (cached_has_bits & 0x00000100u) { ::google::protobuf::internal::WireFormatLite::WriteInt64(7, this->started_at(), output); } @@ -2863,6 +2894,11 @@ void NodeDescriptor::SerializeWithCachedSizes( 10, this->_internal_sql_address(), output); } + if (cached_has_bits & 0x00000080u) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 11, this->_internal_tenant_address(), output); + } + output->WriteRaw(_internal_metadata_.unknown_fields().data(), static_cast(_internal_metadata_.unknown_fields().size())); // @@protoc_insertion_point(serialize_end:cockroach.roachpb.NodeDescriptor) @@ -2927,19 +2963,27 @@ size_t NodeDescriptor::ByteSizeLong() const { *sql_address_); } + if (has_tenant_address()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSize( + *tenant_address_); + } + + } + if (_has_bits_[8 / 32] & 768u) { if (has_started_at()) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::Int64Size( this->started_at()); } - } - if (has_node_id()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->node_id()); - } + if (has_node_id()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->node_id()); + } + } int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -2984,13 +3028,18 @@ void NodeDescriptor::MergeFrom(const NodeDescriptor& from) { mutable_sql_address()->::cockroach::util::UnresolvedAddr::MergeFrom(from.sql_address()); } if (cached_has_bits & 0x00000080u) { + mutable_tenant_address()->::cockroach::util::UnresolvedAddr::MergeFrom(from.tenant_address()); + } + } + if (cached_has_bits & 768u) { + if (cached_has_bits & 0x00000100u) { started_at_ = from.started_at_; } + if (cached_has_bits & 0x00000200u) { + node_id_ = from.node_id_; + } _has_bits_[0] |= cached_has_bits; } - if (cached_has_bits & 0x00000100u) { - set_node_id(from.node_id()); - } } void NodeDescriptor::CopyFrom(const NodeDescriptor& from) { @@ -3020,6 +3069,7 @@ void NodeDescriptor::InternalSwap(NodeDescriptor* other) { swap(locality_, other->locality_); swap(serverversion_, other->serverversion_); swap(sql_address_, other->sql_address_); + swap(tenant_address_, other->tenant_address_); swap(started_at_, other->started_at_); swap(node_id_, other->node_id_); swap(_has_bits_[0], other->_has_bits_[0]); diff --git a/c-deps/libroach/protos/roachpb/metadata.pb.h b/c-deps/libroach/protos/roachpb/metadata.pb.h index 0304ab3e3d1d..51ba9bcca52a 100644 --- a/c-deps/libroach/protos/roachpb/metadata.pb.h +++ b/c-deps/libroach/protos/roachpb/metadata.pb.h @@ -1369,6 +1369,17 @@ class NodeDescriptor : public ::google::protobuf::MessageLite /* @@protoc_insert ::cockroach::util::UnresolvedAddr* mutable_sql_address(); void set_allocated_sql_address(::cockroach::util::UnresolvedAddr* sql_address); + bool has_tenant_address() const; + void clear_tenant_address(); + static const int kTenantAddressFieldNumber = 11; + private: + const ::cockroach::util::UnresolvedAddr& _internal_tenant_address() const; + public: + const ::cockroach::util::UnresolvedAddr& tenant_address() const; + ::cockroach::util::UnresolvedAddr* release_tenant_address(); + ::cockroach::util::UnresolvedAddr* mutable_tenant_address(); + void set_allocated_tenant_address(::cockroach::util::UnresolvedAddr* tenant_address); + bool has_started_at() const; void clear_started_at(); static const int kStartedAtFieldNumber = 7; @@ -1401,6 +1412,8 @@ class NodeDescriptor : public ::google::protobuf::MessageLite /* @@protoc_insert void clear_has_cluster_name(); void set_has_sql_address(); void clear_has_sql_address(); + void set_has_tenant_address(); + void clear_has_tenant_address(); ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; ::google::protobuf::internal::HasBits<1> _has_bits_; @@ -1413,6 +1426,7 @@ class NodeDescriptor : public ::google::protobuf::MessageLite /* @@protoc_insert ::cockroach::roachpb::Locality* locality_; ::cockroach::roachpb::Version* serverversion_; ::cockroach::util::UnresolvedAddr* sql_address_; + ::cockroach::util::UnresolvedAddr* tenant_address_; ::google::protobuf::int64 started_at_; ::google::protobuf::int32 node_id_; friend struct ::protobuf_roachpb_2fmetadata_2eproto::TableStruct; @@ -3257,13 +3271,13 @@ inline void StoreCapacity::set_allocated_writes_per_replica(::cockroach::roachpb // NodeDescriptor inline bool NodeDescriptor::has_node_id() const { - return (_has_bits_[0] & 0x00000100u) != 0; + return (_has_bits_[0] & 0x00000200u) != 0; } inline void NodeDescriptor::set_has_node_id() { - _has_bits_[0] |= 0x00000100u; + _has_bits_[0] |= 0x00000200u; } inline void NodeDescriptor::clear_has_node_id() { - _has_bits_[0] &= ~0x00000100u; + _has_bits_[0] &= ~0x00000200u; } inline void NodeDescriptor::clear_node_id() { node_id_ = 0; @@ -3569,13 +3583,13 @@ inline void NodeDescriptor::set_allocated_build_tag(::std::string* build_tag) { } inline bool NodeDescriptor::has_started_at() const { - return (_has_bits_[0] & 0x00000080u) != 0; + return (_has_bits_[0] & 0x00000100u) != 0; } inline void NodeDescriptor::set_has_started_at() { - _has_bits_[0] |= 0x00000080u; + _has_bits_[0] |= 0x00000100u; } inline void NodeDescriptor::clear_has_started_at() { - _has_bits_[0] &= ~0x00000080u; + _has_bits_[0] &= ~0x00000100u; } inline void NodeDescriptor::clear_started_at() { started_at_ = GOOGLE_LONGLONG(0); @@ -3738,6 +3752,59 @@ inline void NodeDescriptor::set_allocated_sql_address(::cockroach::util::Unresol // @@protoc_insertion_point(field_set_allocated:cockroach.roachpb.NodeDescriptor.sql_address) } +inline bool NodeDescriptor::has_tenant_address() const { + return (_has_bits_[0] & 0x00000080u) != 0; +} +inline void NodeDescriptor::set_has_tenant_address() { + _has_bits_[0] |= 0x00000080u; +} +inline void NodeDescriptor::clear_has_tenant_address() { + _has_bits_[0] &= ~0x00000080u; +} +inline const ::cockroach::util::UnresolvedAddr& NodeDescriptor::_internal_tenant_address() const { + return *tenant_address_; +} +inline const ::cockroach::util::UnresolvedAddr& NodeDescriptor::tenant_address() const { + const ::cockroach::util::UnresolvedAddr* p = tenant_address_; + // @@protoc_insertion_point(field_get:cockroach.roachpb.NodeDescriptor.tenant_address) + return p != NULL ? *p : *reinterpret_cast( + &::cockroach::util::_UnresolvedAddr_default_instance_); +} +inline ::cockroach::util::UnresolvedAddr* NodeDescriptor::release_tenant_address() { + // @@protoc_insertion_point(field_release:cockroach.roachpb.NodeDescriptor.tenant_address) + clear_has_tenant_address(); + ::cockroach::util::UnresolvedAddr* temp = tenant_address_; + tenant_address_ = NULL; + return temp; +} +inline ::cockroach::util::UnresolvedAddr* NodeDescriptor::mutable_tenant_address() { + set_has_tenant_address(); + if (tenant_address_ == NULL) { + auto* p = CreateMaybeMessage<::cockroach::util::UnresolvedAddr>(GetArenaNoVirtual()); + tenant_address_ = p; + } + // @@protoc_insertion_point(field_mutable:cockroach.roachpb.NodeDescriptor.tenant_address) + return tenant_address_; +} +inline void NodeDescriptor::set_allocated_tenant_address(::cockroach::util::UnresolvedAddr* tenant_address) { + ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); + if (message_arena == NULL) { + delete reinterpret_cast< ::google::protobuf::MessageLite*>(tenant_address_); + } + if (tenant_address) { + ::google::protobuf::Arena* submessage_arena = NULL; + if (message_arena != submessage_arena) { + tenant_address = ::google::protobuf::internal::GetOwnedMessage( + message_arena, tenant_address, submessage_arena); + } + set_has_tenant_address(); + } else { + clear_has_tenant_address(); + } + tenant_address_ = tenant_address; + // @@protoc_insertion_point(field_set_allocated:cockroach.roachpb.NodeDescriptor.tenant_address) +} + // ------------------------------------------------------------------- // LocalityAddress diff --git a/pkg/base/addr_validation.go b/pkg/base/addr_validation.go index 8b6cef3d2a53..8b7d8dafc10b 100644 --- a/pkg/base/addr_validation.go +++ b/pkg/base/addr_validation.go @@ -66,6 +66,22 @@ func (cfg *Config) ValidateAddrs(ctx context.Context) error { } cfg.SQLAddr = net.JoinHostPort(sqlHost, sqlPort) + // Validate the tenant advertise address. Use the provided advertise + // addr as default. + advTenantHost, advTenantPort, err := validateAdvertiseAddr(ctx, + cfg.TenantAdvertiseAddr, "--tenant-addr", cfg.TenantAddr, advHost) + if err != nil { + return errors.Wrap(err, "invalid --advertise-tenant-addr") + } + cfg.TenantAdvertiseAddr = net.JoinHostPort(advTenantHost, advTenantPort) + + // Validate the tenant listen address - use the resolved listen addr as default. + tenantHost, tenantPort, err := validateListenAddr(ctx, cfg.TenantAddr, listenHost) + if err != nil { + return errors.Wrap(err, "invalid --tenant-addr") + } + cfg.TenantAddr = net.JoinHostPort(tenantHost, tenantPort) + // Validate the HTTP advertise address. Use the provided advertise // addr as default. advHTTPHost, advHTTPPort, err := validateAdvertiseAddr(ctx, diff --git a/pkg/base/addr_validation_test.go b/pkg/base/addr_validation_test.go index f304cecb6a19..98c58b24dae9 100644 --- a/pkg/base/addr_validation_test.go +++ b/pkg/base/addr_validation_test.go @@ -23,11 +23,15 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/leaktest" ) -type addrs struct{ listen, adv, http, advhttp, sql, advsql string } +type addrs struct{ listen, adv, http, advhttp, sql, advsql, ten, advten string } func (a *addrs) String() string { - return fmt.Sprintf("--listen-addr=%s --advertise-addr=%s --http-addr=%s (http adv: %s) --sql-addr=%s (sql adv: %s)", - a.listen, a.adv, a.http, a.advhttp, a.sql, a.advsql) + return fmt.Sprintf(""+ + "--listen-addr=%s --advertise-addr=%s "+ + "--http-addr=%s (http adv: %s) "+ + "--sql-addr=%s (sql adv: %s) "+ + "--tenant-addr=%s (tenant adv: %s)", + a.listen, a.adv, a.http, a.advhttp, a.sql, a.advsql, a.ten, a.advten) } func TestValidateAddrs(t *testing.T) { @@ -76,84 +80,91 @@ func TestValidateAddrs(t *testing.T) { expected addrs }{ // Common case: no server flags, all defaults. - {addrs{":26257", "", ":8080", "", ":5432", ""}, "", - addrs{":26257", hostname + ":26257", ":8080", hostname + ":8080", ":5432", hostname + ":5432"}}, + {addrs{":26257", "", ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{":26257", hostname + ":26257", ":8080", hostname + ":8080", ":5432", hostname + ":5432", ":26258", hostname + ":26258"}}, // Another common case: --listen-addr= - {addrs{hostname + ":26257", "", ":8080", "", ":5432", ""}, "", - addrs{hostAddr + ":26257", hostname + ":26257", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432"}}, + {addrs{hostname + ":26257", "", ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{hostAddr + ":26257", hostname + ":26257", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432", hostAddr + ":26258", hostname + ":26258"}}, // Another common case: --listen-addr=localhost - {addrs{"localhost:26257", "", ":8080", "", ":5432", ""}, "", - addrs{localAddr + ":26257", "localhost:26257", localAddr + ":8080", "localhost:8080", localAddr + ":5432", "localhost:5432"}}, + {addrs{"localhost:26257", "", ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{localAddr + ":26257", "localhost:26257", localAddr + ":8080", "localhost:8080", localAddr + ":5432", "localhost:5432", localAddr + ":26258", "localhost:26258"}}, // Correct use: --listen-addr= --advertise-host= - {addrs{hostAddr + ":26257", hostname + ":", ":8080", "", ":5432", ""}, "", - addrs{hostAddr + ":26257", hostname + ":26257", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432"}}, + {addrs{hostAddr + ":26257", hostname + ":", ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{hostAddr + ":26257", hostname + ":26257", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432", hostAddr + ":26258", hostname + ":26258"}}, // Explicit port number in advertise addr. - {addrs{hostAddr + ":26257", hostname + ":12345", ":8080", "", ":5432", ""}, "", - addrs{hostAddr + ":26257", hostname + ":12345", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432"}}, + {addrs{hostAddr + ":26257", hostname + ":12345", ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{hostAddr + ":26257", hostname + ":12345", hostAddr + ":8080", hostname + ":8080", hostAddr + ":5432", hostname + ":5432", hostAddr + ":26258", hostname + ":26258"}}, // Use a non-numeric port number. - {addrs{":postgresql", "", ":http", "", ":postgresql", ""}, "", - addrs{":5432", hostname + ":5432", ":80", hostname + ":80", ":5432", hostname + ":5432"}}, + {addrs{":postgresql", "", ":http", "", ":postgresql", "", ":postgresql", ""}, "", + addrs{":5432", hostname + ":5432", ":80", hostname + ":80", ":5432", hostname + ":5432", ":5432", hostname + ":5432"}}, // Make HTTP local only. - {addrs{":26257", "", "localhost:8080", "", ":5432", ""}, "", - addrs{":26257", hostname + ":26257", localAddr + ":8080", "localhost:8080", ":5432", hostname + ":5432"}}, + {addrs{":26257", "", "localhost:8080", "", ":5432", "", ":26258", ""}, "", + addrs{":26257", hostname + ":26257", localAddr + ":8080", "localhost:8080", ":5432", hostname + ":5432", ":26258", hostname + ":26258"}}, // Local server but public HTTP. - {addrs{"localhost:26257", "", hostname + ":8080", "", ":5432", ""}, "", - addrs{localAddr + ":26257", "localhost:26257", hostAddr + ":8080", hostname + ":8080", localAddr + ":5432", "localhost:5432"}}, + {addrs{"localhost:26257", "", hostname + ":8080", "", ":5432", "", ":26258", ""}, "", + addrs{localAddr + ":26257", "localhost:26257", hostAddr + ":8080", hostname + ":8080", localAddr + ":5432", "localhost:5432", localAddr + ":26258", "localhost:26258"}}, - // Make SQL local only. - {addrs{":26257", "", ":8080", "", "localhost:5432", ""}, "", - addrs{":26257", hostname + ":26257", ":8080", hostname + ":8080", localAddr + ":5432", "localhost:5432"}}, + // Make SQL and tenant local only. + {addrs{":26257", "", ":8080", "", "localhost:5432", "", "localhost:26258", ""}, "", + addrs{":26257", hostname + ":26257", ":8080", hostname + ":8080", localAddr + ":5432", "localhost:5432", localAddr + ":26258", "localhost:26258"}}, // Not-unreasonable case: addresses set empty. Means using port 0. - {addrs{"", "", "", "", "", ""}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", "", "", "", "", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, // A colon means "all-addr, auto-port". - {addrs{":", "", "", "", "", ""}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, - {addrs{"", ":", "", "", "", ""}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, - {addrs{"", "", ":", "", "", ""}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, - {addrs{"", "", "", "", ":", ""}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, - {addrs{"", "", "", "", "", ":"}, "", - addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{":", "", "", "", "", "", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", ":", "", "", "", "", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", ":", "", "", "", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", "", "", ":", "", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", "", "", "", ":", "", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", "", "", "", "", ":", ""}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, + {addrs{"", "", "", "", "", "", "", ":"}, "", + addrs{":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0", ":0", hostname + ":0"}}, // Advertise port 0 means reuse listen port. We don't // auto-allocate ports for advertised addresses. - {addrs{":12345", ":0", "", "", ":5432", ""}, "", - addrs{":12345", hostname + ":12345", ":0", hostname + ":0", ":5432", hostname + ":5432"}}, - {addrs{":12345", "", "", "", ":5432", ":0"}, "", - addrs{":12345", hostname + ":12345", ":0", hostname + ":0", ":5432", hostname + ":5432"}}, + {addrs{":12345", ":0", "", "", ":5432", "", ":26258", ""}, "", + addrs{":12345", hostname + ":12345", ":0", hostname + ":0", ":5432", hostname + ":5432", ":26258", hostname + ":26258"}}, + {addrs{":12345", "", "", "", ":5432", ":0", ":26258", ":0"}, "", + addrs{":12345", hostname + ":12345", ":0", hostname + ":0", ":5432", hostname + ":5432", ":26258", hostname + ":26258"}}, // Expected errors. // Missing port number. - {addrs{"localhost", "", "", "", "", ""}, "invalid --listen-addr.*missing port in address", addrs{}}, - {addrs{":26257", "", "localhost", "", "", ""}, "invalid --http-addr.*missing port in address", addrs{}}, + {addrs{"localhost", "", "", "", "", "", "", ""}, "invalid --listen-addr.*missing port in address", addrs{}}, + {addrs{":26257", "", "localhost", "", "", "", "", ""}, "invalid --http-addr.*missing port in address", addrs{}}, // Invalid port number. - {addrs{"localhost:-1231", "", "", "", "", ""}, "invalid port", addrs{}}, - {addrs{"localhost:nonexistent", "", "", "", "", ""}, portExpectedErr, addrs{}}, + {addrs{"localhost:-1231", "", "", "", "", "", "", ""}, "invalid port", addrs{}}, + {addrs{"localhost:nonexistent", "", "", "", "", "", "", ""}, portExpectedErr, addrs{}}, // Invalid address. - {addrs{"nonexistent.example.com:26257", "", "", "", "", ""}, "no such host", addrs{}}, - {addrs{"333.333.333.333:26257", "", "", "", "", ""}, "no such host", addrs{}}, + {addrs{"nonexistent.example.com:26257", "", "", "", "", "", "", ""}, "no such host", addrs{}}, + {addrs{"333.333.333.333:26257", "", "", "", "", "", "", ""}, "no such host", addrs{}}, } for i, test := range testData { t.Run(fmt.Sprintf("%d/%s", i, test.in), func(t *testing.T) { cfg := base.Config{ - Addr: test.in.listen, - AdvertiseAddr: test.in.adv, - HTTPAddr: test.in.http, - SQLAddr: test.in.sql, - SQLAdvertiseAddr: test.in.advsql, + Addr: test.in.listen, + AdvertiseAddr: test.in.adv, + HTTPAddr: test.in.http, + HTTPAdvertiseAddr: test.in.advhttp, + SQLAddr: test.in.sql, + SQLAdvertiseAddr: test.in.advsql, + TenantAddr: test.in.ten, + TenantAdvertiseAddr: test.in.advten, } if err := cfg.ValidateAddrs(context.Background()); err != nil { @@ -166,7 +177,16 @@ func TestValidateAddrs(t *testing.T) { t.Fatalf("expected error %q, got success", test.expectedErr) } - got := addrs{cfg.Addr, cfg.AdvertiseAddr, cfg.HTTPAddr, cfg.HTTPAdvertiseAddr, cfg.SQLAddr, cfg.SQLAdvertiseAddr} + got := addrs{ + listen: cfg.Addr, + adv: cfg.AdvertiseAddr, + http: cfg.HTTPAddr, + advhttp: cfg.HTTPAdvertiseAddr, + sql: cfg.SQLAddr, + advsql: cfg.SQLAdvertiseAddr, + ten: cfg.TenantAddr, + advten: cfg.TenantAdvertiseAddr, + } gotStr := got.String() expStr := test.expected.String() diff --git a/pkg/base/config.go b/pkg/base/config.go index 7783c2dd9d11..120fb5dd2bd6 100644 --- a/pkg/base/config.go +++ b/pkg/base/config.go @@ -41,9 +41,10 @@ const ( DefaultHTTPPort = "8080" // NB: net.JoinHostPort is not a constant. - defaultAddr = ":" + DefaultPort - defaultHTTPAddr = ":" + DefaultHTTPPort - defaultSQLAddr = ":" + DefaultPort + defaultAddr = ":" + DefaultPort + defaultSQLAddr = ":" + DefaultPort + defaultTenantAddr = ":" + DefaultPort + defaultHTTPAddr = ":" + DefaultHTTPPort // NetworkTimeout is the timeout used for network operations. NetworkTimeout = 3 * time.Second @@ -190,6 +191,18 @@ type Config struct { // This is computed from SQLAddr if specified otherwise Addr. SQLAdvertiseAddr string + // SplitListenTenant indicates whether to listen for tenant + // KV clients on a separate address from RPC requests. + SplitListenTenant bool + + // TenantAddr is the configured tenant KV listen address. + // This is used if SplitListenTenant is set to true. + TenantAddr string + + // TenantAdvertiseAddr is the advertised tenant KV address. + // This is computed from TenantAddr if specified otherwise Addr. + TenantAdvertiseAddr string + // HTTPAddr is the configured HTTP listen address. HTTPAddr string @@ -241,6 +254,9 @@ func (cfg *Config) InitDefaults() { cfg.SplitListenSQL = false cfg.SQLAddr = defaultSQLAddr cfg.SQLAdvertiseAddr = cfg.SQLAddr + cfg.SplitListenTenant = false + cfg.TenantAddr = defaultTenantAddr + cfg.TenantAdvertiseAddr = cfg.TenantAddr cfg.SSLCertsDir = DefaultCertsDirectory cfg.RPCHeartbeatInterval = defaultRPCHeartbeatInterval cfg.ClusterName = "" diff --git a/pkg/base/test_server_args.go b/pkg/base/test_server_args.go index 4cd9e7dfb3b6..8a7152010798 100644 --- a/pkg/base/test_server_args.go +++ b/pkg/base/test_server_args.go @@ -48,6 +48,8 @@ type TestServerArgs struct { Addr string // SQLAddr (if nonempty) is the SQL address to use for the test server. SQLAddr string + // TenantAddr (if nonempty) is the tenant KV address to use for the test server. + TenantAddr string // HTTPAddr (if nonempty) is the HTTP address to use for the test server. HTTPAddr string // DisableTLSForHTTP if set, disables TLS for the HTTP interface. diff --git a/pkg/cli/cliflags/flags.go b/pkg/cli/cliflags/flags.go index fe8048307d63..2c62a9163a29 100644 --- a/pkg/cli/cliflags/flags.go +++ b/pkg/cli/cliflags/flags.go @@ -454,6 +454,45 @@ The port number should be the same as in --listen-addr unless port forwarding is set up on an intermediate firewall/router.`, } + AdvertiseHost = FlagInfo{ + Name: "advertise-host", + Description: `Alias for --advertise-addr. Deprecated.`, + } + + AdvertisePort = FlagInfo{ + Name: "advertise-port", + Description: `Deprecated. Use --advertise-addr=:.`, + } + + ListenSQLAddr = FlagInfo{ + Name: "sql-addr", + Description: ` +The hostname or IP address to bind to for SQL clients, for example +--sql-addr=myhost:26257 or --sql-addr=:26257 (listen on all interfaces). +If left unspecified, the address specified by --listen-addr will be +used for both RPC and SQL connections. +
+
+
+If specified but the address part is omitted, the address part +defaults to the address part of --listen-addr. +If specified but the port number is omitted, the port number +defaults to 26257. +
+
+
+To actually use separate bindings, it is recommended to specify +both flags and use a different port number via --listen-addr, for +example --sql-addr=:26257 --listen-addr=:26258. Ensure that +--join is set accordingly on other nodes. It is also possible +to use the same port number but separate host addresses. +
+
+
+An IPv6 address can also be specified with the notation [...], for +example [::1]:26257 or [fe80::f6f2:::]:26257.`, + } + SQLAdvertiseAddr = FlagInfo{ Name: "advertise-sql-addr", Description: ` @@ -479,43 +518,57 @@ The port number should be the same as in --sql-addr unless port forwarding is set up on an intermediate firewall/router.`, } - AdvertiseHost = FlagInfo{ - Name: "advertise-host", - Description: `Alias for --advertise-addr. Deprecated.`, - } + ListenTenantAddr = FlagInfo{ + Name: "tenant-addr", + Description: ` +The hostname or IP address to bind to for tenant KV clients, for example +--tenant-addr=myhost:26257 or --tenant-addr=:26257 (listen on all +interfaces). If left unspecified, the address specified by --listen-addr +will be used for both RPC and SQL connections. +
 
-	AdvertisePort = FlagInfo{
-		Name:        "advertise-port",
-		Description: `Deprecated. Use --advertise-addr=:.`,
+
+If specified but the address part is omitted, the address part defaults +to the address part of --listen-addr. If specified but the port number +is omitted, the port number defaults to 26257. +
+
+
+To actually use separate bindings, it is recommended to specify both +flags and use a different port number via --listen-addr, for example +--tenant-addr=:36257 --listen-addr=:26257. Ensure that --join is set +accordingly on other nodes. It is also possible to use the same port +number but separate host addresses. +
+
+
+An IPv6 address can also be specified with the notation [...], for +example [::1]:26257 or [fe80::f6f2:::]:26257.`, } - ListenSQLAddr = FlagInfo{ - Name: "sql-addr", + TenantAdvertiseAddr = FlagInfo{ + Name: "advertise-tenant-addr", Description: ` -The hostname or IP address to bind to for SQL clients, for example ---sql-addr=myhost:26257 or --sql-addr=:26257 (listen on all interfaces). -If left unspecified, the address specified by --listen-addr will be -used for both RPC and SQL connections. +The address/hostname and port to advertise to tenant SQL nodes for +tenant KV communication. It must resolve and be routable from other +nodes in the cluster.
-If specified but the address part is omitted, the address part -defaults to the address part of --listen-addr. -If specified but the port number is omitted, the port -number defaults to 26257. +If left unspecified, it defaults to the setting of --tenant-addr. If the +flag is provided but either the address part or the port part is left +unspecified, that particular part defaults to the same part in +--tenant-addr.
-To actually use separate bindings, it is recommended to specify -both flags and use a different port number via --listen-addr, for -example --sql-addr=:26257 --listen-addr=:26258. Ensure that ---join is set accordingly on other nodes. It is also possible -to use the same port number but separate host addresses. +An IPv6 address can also be specified with the notation [...], for +example [::1]:26257 or [fe80::f6f2:::]:26257.
-An IPv6 address can also be specified with the notation [...], for -example [::1]:26257 or [fe80::f6f2:::]:26257.`, +The port number should be the same as in --tenant-addr unless port +forwarding is set up on an intermediate firewall/router.`, } ListenHTTPAddr = FlagInfo{ diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go index df2d2c5cbef4..5437fc09669d 100644 --- a/pkg/cli/flags.go +++ b/pkg/cli/flags.go @@ -50,6 +50,8 @@ var serverListenPort, serverSocketDir string var serverAdvertiseAddr, serverAdvertisePort string var serverSQLAddr, serverSQLPort string var serverSQLAdvertiseAddr, serverSQLAdvertisePort string +var serverTenantAddr, serverTenantPort string +var serverTenantAdvertiseAddr, serverTenantAdvertisePort string var serverHTTPAddr, serverHTTPPort string var localityAdvertiseHosts localityList var sqlAuditLogDir log.DirName @@ -70,6 +72,11 @@ func initPreFlagsDefaults() { serverSQLAdvertiseAddr = "" serverSQLAdvertisePort = "" + serverTenantAddr = "" + serverTenantPort = "" + serverTenantAdvertiseAddr = "" + serverTenantAdvertisePort = "" + serverHTTPAddr = "" serverHTTPPort = base.DefaultHTTPPort @@ -343,6 +350,8 @@ func init() { varFlag(f, addrSetter{&serverAdvertiseAddr, &serverAdvertisePort}, cliflags.AdvertiseAddr) varFlag(f, addrSetter{&serverSQLAddr, &serverSQLPort}, cliflags.ListenSQLAddr) varFlag(f, addrSetter{&serverSQLAdvertiseAddr, &serverSQLAdvertisePort}, cliflags.SQLAdvertiseAddr) + varFlag(f, addrSetter{&serverTenantAddr, &serverTenantPort}, cliflags.ListenTenantAddr) + varFlag(f, addrSetter{&serverTenantAdvertiseAddr, &serverTenantAdvertisePort}, cliflags.TenantAdvertiseAddr) varFlag(f, addrSetter{&serverHTTPAddr, &serverHTTPPort}, cliflags.ListenHTTPAddr) stringFlag(f, &serverSocketDir, cliflags.SocketDir) // --socket is deprecated as of 20.1. @@ -351,6 +360,10 @@ func init() { _ = f.MarkDeprecated(cliflags.Socket.Name, "use the --socket-dir and --listen-addr flags instead") boolFlag(f, &startCtx.unencryptedLocalhostHTTP, cliflags.UnencryptedLocalhostHTTP) + // Hide tenant-related flags. + _ = f.MarkHidden(cliflags.ListenTenantAddr.Name) + _ = f.MarkHidden(cliflags.TenantAdvertiseAddr.Name) + // Backward-compatibility flags. // These are deprecated but until we have qualitatively new @@ -964,6 +977,36 @@ func extraServerFlagInit(cmd *cobra.Command) error { } serverCfg.SQLAdvertiseAddr = net.JoinHostPort(serverSQLAdvertiseAddr, serverSQLAdvertisePort) + // Fill in the defaults for --tenant-addr. + if serverTenantAddr == "" { + serverTenantAddr = startCtx.serverListenAddr + } + if serverTenantPort == "" { + serverTenantPort = serverListenPort + } + serverCfg.TenantAddr = net.JoinHostPort(serverTenantAddr, serverTenantPort) + // NOTE: multi-tenancy commands don't register this flag. + if f := fs.Lookup(cliflags.ListenTenantAddr.Name); f != nil { + serverCfg.SplitListenTenant = f.Changed + } + + // Fill in the defaults for --advertise-tenant-addr, if the flag exists on `cmd`. + if serverTenantAdvertiseAddr == "" { + if advSpecified { + serverTenantAdvertiseAddr = serverAdvertiseAddr + } else { + serverTenantAdvertiseAddr = serverTenantAddr + } + } + if serverTenantAdvertisePort == "" { + if advSpecified && !serverCfg.SplitListenTenant { + serverTenantAdvertisePort = serverAdvertisePort + } else { + serverTenantAdvertisePort = serverTenantPort + } + } + serverCfg.TenantAdvertiseAddr = net.JoinHostPort(serverTenantAdvertiseAddr, serverTenantAdvertisePort) + // Fill in the defaults for --http-addr. if serverHTTPAddr == "" { serverHTTPAddr = startCtx.serverListenAddr diff --git a/pkg/cli/flags_test.go b/pkg/cli/flags_test.go index 03949a291b9a..397f0a96a00c 100644 --- a/pkg/cli/flags_test.go +++ b/pkg/cli/flags_test.go @@ -437,128 +437,209 @@ func TestServerConnSettings(t *testing.T) { expectedAdvertiseAddr string expSQLAddr string expSQLAdvAddr string + expTenantAddr string + expTenantAdvAddr string }{ {[]string{"start"}, ":" + base.DefaultPort, ":" + base.DefaultPort, ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "127.0.0.1"}, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "192.168.0.111"}, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", ":"}, ":", ":", ":", ":", + ":", ":", }, {[]string{"start", "--listen-addr", "127.0.0.1:"}, "127.0.0.1:", "127.0.0.1:", "127.0.0.1:", "127.0.0.1:", + "127.0.0.1:", "127.0.0.1:", }, {[]string{"start", "--listen-addr", ":12345"}, ":12345", ":12345", ":12345", ":12345", + ":12345", ":12345", }, {[]string{"start", "--listen-addr", "127.0.0.1:12345"}, "127.0.0.1:12345", "127.0.0.1:12345", "127.0.0.1:12345", "127.0.0.1:12345", + "127.0.0.1:12345", "127.0.0.1:12345", }, {[]string{"start", "--listen-addr", "[::1]"}, "[::1]:" + base.DefaultPort, "[::1]:" + base.DefaultPort, "[::1]:" + base.DefaultPort, "[::1]:" + base.DefaultPort, + "[::1]:" + base.DefaultPort, "[::1]:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "[::1]:12345"}, "[::1]:12345", "[::1]:12345", "[::1]:12345", "[::1]:12345", + "[::1]:12345", "[::1]:12345", }, {[]string{"start", "--listen-addr", "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]"}, "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, + "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, "[2622:6221:e663:4922:fc2b:788b:fadd:7b48]:" + base.DefaultPort, }, // confirm hostnames will work {[]string{"start", "--listen-addr", "my.host.name"}, "my.host.name:" + base.DefaultPort, "my.host.name:" + base.DefaultPort, "my.host.name:" + base.DefaultPort, "my.host.name:" + base.DefaultPort, + "my.host.name:" + base.DefaultPort, "my.host.name:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "myhostname"}, "myhostname:" + base.DefaultPort, "myhostname:" + base.DefaultPort, "myhostname:" + base.DefaultPort, "myhostname:" + base.DefaultPort, + "myhostname:" + base.DefaultPort, "myhostname:" + base.DefaultPort, }, // SQL address override. {[]string{"start", "--sql-addr", "127.0.0.1"}, ":" + base.DefaultPort, ":" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, }, {[]string{"start", "--sql-addr", ":1234"}, ":" + base.DefaultPort, ":" + base.DefaultPort, ":1234", ":1234", + ":" + base.DefaultPort, ":" + base.DefaultPort, }, {[]string{"start", "--sql-addr", "127.0.0.1:1234"}, ":" + base.DefaultPort, ":" + base.DefaultPort, "127.0.0.1:1234", "127.0.0.1:1234", + ":" + base.DefaultPort, ":" + base.DefaultPort, }, {[]string{"start", "--sql-addr", "[::2]"}, ":" + base.DefaultPort, ":" + base.DefaultPort, "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, }, {[]string{"start", "--sql-addr", "[::2]:1234"}, ":" + base.DefaultPort, ":" + base.DefaultPort, "[::2]:1234", "[::2]:1234", + ":" + base.DefaultPort, ":" + base.DefaultPort, }, // Configuring the components of the SQL address separately. {[]string{"start", "--listen-addr", "127.0.0.1", "--sql-addr", "127.0.0.2"}, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.2:" + base.DefaultPort, "127.0.0.2:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "127.0.0.1", "--sql-addr", ":1234"}, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:1234", "127.0.0.1:1234", + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "127.0.0.1", "--sql-addr", "127.0.0.2:1234"}, "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "127.0.0.2:1234", "127.0.0.2:1234", + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "[::2]", "--sql-addr", ":1234"}, "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, - "[::2]:1234", "[::2]:1234"}, + "[::2]:1234", "[::2]:1234", + "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, + }, + + // Tenant address override. + {[]string{"start", "--tenant-addr", "127.0.0.1"}, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + }, + {[]string{"start", "--tenant-addr", ":1234"}, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":1234", ":1234", + }, + {[]string{"start", "--tenant-addr", "127.0.0.1:1234"}, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, + "127.0.0.1:1234", "127.0.0.1:1234", + }, + {[]string{"start", "--tenant-addr", "[::2]"}, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, + "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, + }, + {[]string{"start", "--tenant-addr", "[::2]:1234"}, + ":" + base.DefaultPort, ":" + base.DefaultPort, + ":" + base.DefaultPort, ":" + base.DefaultPort, + "[::2]:1234", "[::2]:1234", + }, + + // Configuring the components of the Tenant address separately. + {[]string{"start", "--listen-addr", "127.0.0.1", "--tenant-addr", "127.0.0.2"}, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.2:" + base.DefaultPort, "127.0.0.2:" + base.DefaultPort, + }, + {[]string{"start", "--listen-addr", "127.0.0.1", "--tenant-addr", ":1234"}, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.1:1234", "127.0.0.1:1234", + }, + {[]string{"start", "--listen-addr", "127.0.0.1", "--tenant-addr", "127.0.0.2:1234"}, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, + "127.0.0.2:1234", "127.0.0.2:1234", + }, + {[]string{"start", "--listen-addr", "[::2]", "--tenant-addr", ":1234"}, + "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, + "[::2]:" + base.DefaultPort, "[::2]:" + base.DefaultPort, + "[::2]:1234", "[::2]:1234", + }, // --advertise-addr overrides. {[]string{"start", "--advertise-addr", "192.168.0.111"}, ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, {[]string{"start", "--advertise-addr", "192.168.0.111:12345"}, ":" + base.DefaultPort, "192.168.0.111:12345", ":" + base.DefaultPort, "192.168.0.111:12345", + ":" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111"}, "127.0.0.1:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "127.0.0.1:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + "127.0.0.1:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, {[]string{"start", "--listen-addr", "127.0.0.1:12345", "--advertise-addr", "192.168.0.111"}, "127.0.0.1:12345", "192.168.0.111:12345", "127.0.0.1:12345", "192.168.0.111:12345", + "127.0.0.1:12345", "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111:12345"}, "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", + "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1:54321", "--advertise-addr", "192.168.0.111:12345"}, "127.0.0.1:54321", "192.168.0.111:12345", "127.0.0.1:54321", "192.168.0.111:12345", + "127.0.0.1:54321", "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--listen-addr", ":12345"}, ":12345", "192.168.0.111:12345", ":12345", "192.168.0.111:12345", + ":12345", "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111:12345", "--listen-addr", ":54321"}, ":54321", "192.168.0.111:12345", ":54321", "192.168.0.111:12345", + ":54321", "192.168.0.111:12345", }, // Show that if the SQL address does not have a name default, its @@ -566,6 +647,7 @@ func TestServerConnSettings(t *testing.T) { {[]string{"start", "--advertise-addr", "192.168.0.111:12345", "--sql-addr", ":54321"}, ":" + base.DefaultPort, "192.168.0.111:12345", ":54321", "192.168.0.111:54321", + ":" + base.DefaultPort, "192.168.0.111:12345", }, // Show that if the SQL address is overridden, its advertised form picks the @@ -573,68 +655,84 @@ func TestServerConnSettings(t *testing.T) { {[]string{"start", "--advertise-addr", "192.168.0.111:12345", "--sql-addr", "127.0.0.1:54321"}, ":" + base.DefaultPort, "192.168.0.111:12345", "127.0.0.1:54321", "192.168.0.111:54321", + ":" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111:12345", "--sql-addr", "127.0.0.1"}, ":" + base.DefaultPort, "192.168.0.111:12345", "127.0.0.1:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + ":" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--sql-addr", "127.0.0.1:12345"}, ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "127.0.0.1:12345", "192.168.0.111:12345", + ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, // Backward-compatibility flag combinations. {[]string{"start", "--host", "192.168.0.111"}, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + "192.168.0.111:" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, {[]string{"start", "--port", "12345"}, ":12345", ":12345", ":12345", ":12345", + ":12345", ":12345", }, {[]string{"start", "--advertise-host", "192.168.0.111"}, ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, + ":" + base.DefaultPort, "192.168.0.111:" + base.DefaultPort, }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--advertise-port", "12345"}, ":" + base.DefaultPort, "192.168.0.111:12345", ":" + base.DefaultPort, "192.168.0.111:12345", + ":" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--port", "12345"}, "127.0.0.1:12345", "127.0.0.1:12345", "127.0.0.1:12345", "127.0.0.1:12345", + "127.0.0.1:12345", "127.0.0.1:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1:12345", "--port", "55555"}, "127.0.0.1:55555", "127.0.0.1:55555", "127.0.0.1:55555", "127.0.0.1:55555", + "127.0.0.1:55555", "127.0.0.1:55555", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111", "--port", "12345"}, "127.0.0.1:12345", "192.168.0.111:12345", "127.0.0.1:12345", "192.168.0.111:12345", + "127.0.0.1:12345", "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111", "--port", "12345"}, "127.0.0.1:12345", "192.168.0.111:12345", "127.0.0.1:12345", "192.168.0.111:12345", + "127.0.0.1:12345", "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111", "--advertise-port", "12345"}, "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", + "127.0.0.1:" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--listen-addr", "127.0.0.1", "--advertise-addr", "192.168.0.111", "--port", "54321", "--advertise-port", "12345"}, "127.0.0.1:54321", "192.168.0.111:12345", "127.0.0.1:54321", "192.168.0.111:12345", + "127.0.0.1:54321", "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--port", "12345"}, ":12345", "192.168.0.111:12345", ":12345", "192.168.0.111:12345", + ":12345", "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--advertise-port", "12345"}, ":" + base.DefaultPort, "192.168.0.111:12345", ":" + base.DefaultPort, "192.168.0.111:12345", + ":" + base.DefaultPort, "192.168.0.111:12345", }, {[]string{"start", "--advertise-addr", "192.168.0.111", "--port", "54321", "--advertise-port", "12345"}, ":54321", "192.168.0.111:12345", ":54321", "192.168.0.111:12345", + ":54321", "192.168.0.111:12345", }, } @@ -658,15 +756,21 @@ func TestServerConnSettings(t *testing.T) { } wantSQLSplit := false + wantTenantSplit := false for _, r := range td.args { - if r == "--sql-addr" { + switch r { + case "--sql-addr": wantSQLSplit = true - break + case "--tenant-addr": + wantTenantSplit = true } } if wantSQLSplit != serverCfg.SplitListenSQL { t.Errorf("%d. expected combined RPC/SQL listen = %v, found %v", i, wantSQLSplit, serverCfg.SplitListenSQL) } + if wantTenantSplit != serverCfg.SplitListenTenant { + t.Errorf("%d. expected split tenant KV listen = %v, found %v", i, wantTenantSplit, serverCfg.SplitListenTenant) + } if td.expSQLAddr != serverCfg.SQLAddr { t.Errorf("%d. serverCfg.SQLAddr expected '%s', got '%s'. td.args was '%#v'.", diff --git a/pkg/cli/zip.go b/pkg/cli/zip.go index 906cec305172..5189ae41cde4 100644 --- a/pkg/cli/zip.go +++ b/pkg/cli/zip.go @@ -477,12 +477,7 @@ func runDebugZip(cmd *cobra.Command, args []string) (retErr error) { // not work and if it doesn't, we let the invalid curSQLConn get // used anyway so that anything that does *not* need it will // still happen. - sqlAddr := node.Desc.SQLAddress - if sqlAddr.IsEmpty() { - // No SQL address: either a pre-19.2 node, or same address for both - // SQL and RPC. - sqlAddr = node.Desc.Address - } + sqlAddr := node.Desc.CheckedSQLAddress() curSQLConn := guessNodeURL(sqlConn.url, sqlAddr.AddressField) if err := z.createJSON(prefix+"/status.json", node); err != nil { return err diff --git a/pkg/roachpb/metadata.go b/pkg/roachpb/metadata.go index 61ce06b928d9..6846c355f70c 100644 --- a/pkg/roachpb/metadata.go +++ b/pkg/roachpb/metadata.go @@ -470,6 +470,29 @@ func (n *NodeDescriptor) AddressForLocality(loc Locality) *util.UnresolvedAddr { return &n.Address } +// CheckedSQLAddress returns the value of SQLAddress if set. If not, either +// because the receiver is a pre-19.2 node, or because it is using the same +// address for both SQL and RPC, the Address is returned. +func (n *NodeDescriptor) CheckedSQLAddress() *util.UnresolvedAddr { + if n.SQLAddress.IsEmpty() { + return &n.Address + } + return &n.SQLAddress +} + +// CheckedTenantAddress returns the value of TenantAddress if set. If not, +// either because the receiver is a pre-20.2 node, or because it is using the +// same address for both Tenant KV and RPC, the Address is returned. +func (n *NodeDescriptor) CheckedTenantAddress() *util.UnresolvedAddr { + if n.TenantAddress.IsEmpty() { + return &n.Address + } + return &n.TenantAddress +} + +// Silence unused warning. +var _ = (*NodeDescriptor).CheckedTenantAddress + // String returns a string representation of the Tier. func (t Tier) String() string { return fmt.Sprintf("%s=%s", t.Key, t.Value) diff --git a/pkg/roachpb/metadata.pb.go b/pkg/roachpb/metadata.pb.go index 123baca0f1e5..c457aab9b1ce 100644 --- a/pkg/roachpb/metadata.pb.go +++ b/pkg/roachpb/metadata.pb.go @@ -112,7 +112,7 @@ func (x *ReplicaType) UnmarshalJSON(data []byte) error { return nil } func (ReplicaType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{0} + return fileDescriptor_metadata_681e86602fa51f2e, []int{0} } // Attributes specifies a list of arbitrary strings describing @@ -124,7 +124,7 @@ type Attributes struct { func (m *Attributes) Reset() { *m = Attributes{} } func (*Attributes) ProtoMessage() {} func (*Attributes) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{0} + return fileDescriptor_metadata_681e86602fa51f2e, []int{0} } func (m *Attributes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -158,7 +158,7 @@ type ReplicationTarget struct { func (m *ReplicationTarget) Reset() { *m = ReplicationTarget{} } func (*ReplicationTarget) ProtoMessage() {} func (*ReplicationTarget) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{1} + return fileDescriptor_metadata_681e86602fa51f2e, []int{1} } func (m *ReplicationTarget) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -203,7 +203,7 @@ type ReplicaDescriptor struct { func (m *ReplicaDescriptor) Reset() { *m = ReplicaDescriptor{} } func (*ReplicaDescriptor) ProtoMessage() {} func (*ReplicaDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{2} + return fileDescriptor_metadata_681e86602fa51f2e, []int{2} } func (m *ReplicaDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -238,7 +238,7 @@ func (m *ReplicaIdent) Reset() { *m = ReplicaIdent{} } func (m *ReplicaIdent) String() string { return proto.CompactTextString(m) } func (*ReplicaIdent) ProtoMessage() {} func (*ReplicaIdent) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{3} + return fileDescriptor_metadata_681e86602fa51f2e, []int{3} } func (m *ReplicaIdent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -370,7 +370,7 @@ type RangeDescriptor struct { func (m *RangeDescriptor) Reset() { *m = RangeDescriptor{} } func (*RangeDescriptor) ProtoMessage() {} func (*RangeDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{4} + return fileDescriptor_metadata_681e86602fa51f2e, []int{4} } func (m *RangeDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -409,7 +409,7 @@ type Percentiles struct { func (m *Percentiles) Reset() { *m = Percentiles{} } func (*Percentiles) ProtoMessage() {} func (*Percentiles) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{5} + return fileDescriptor_metadata_681e86602fa51f2e, []int{5} } func (m *Percentiles) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -471,7 +471,7 @@ type StoreCapacity struct { func (m *StoreCapacity) Reset() { *m = StoreCapacity{} } func (*StoreCapacity) ProtoMessage() {} func (*StoreCapacity) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{6} + return fileDescriptor_metadata_681e86602fa51f2e, []int{6} } func (m *StoreCapacity) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,13 +510,16 @@ type NodeDescriptor struct { // The SQL address. If empty, indicates that the base address field // is also used to accept SQL connections. SQLAddress util.UnresolvedAddr `protobuf:"bytes,10,opt,name=sql_address,json=sqlAddress" json:"sql_address"` + // The tenant KV address. If empty, indicates that the base address + // field is also used to accept tenant KV connections. + TenantAddress util.UnresolvedAddr `protobuf:"bytes,11,opt,name=tenant_address,json=tenantAddress" json:"tenant_address"` } func (m *NodeDescriptor) Reset() { *m = NodeDescriptor{} } func (m *NodeDescriptor) String() string { return proto.CompactTextString(m) } func (*NodeDescriptor) ProtoMessage() {} func (*NodeDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{7} + return fileDescriptor_metadata_681e86602fa51f2e, []int{7} } func (m *NodeDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -552,7 +555,7 @@ func (m *LocalityAddress) Reset() { *m = LocalityAddress{} } func (m *LocalityAddress) String() string { return proto.CompactTextString(m) } func (*LocalityAddress) ProtoMessage() {} func (*LocalityAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{8} + return fileDescriptor_metadata_681e86602fa51f2e, []int{8} } func (m *LocalityAddress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -590,7 +593,7 @@ func (m *StoreDescriptor) Reset() { *m = StoreDescriptor{} } func (m *StoreDescriptor) String() string { return proto.CompactTextString(m) } func (*StoreDescriptor) ProtoMessage() {} func (*StoreDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{9} + return fileDescriptor_metadata_681e86602fa51f2e, []int{9} } func (m *StoreDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -627,7 +630,7 @@ func (m *StoreDeadReplicas) Reset() { *m = StoreDeadReplicas{} } func (m *StoreDeadReplicas) String() string { return proto.CompactTextString(m) } func (*StoreDeadReplicas) ProtoMessage() {} func (*StoreDeadReplicas) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{10} + return fileDescriptor_metadata_681e86602fa51f2e, []int{10} } func (m *StoreDeadReplicas) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -661,7 +664,7 @@ type Locality struct { func (m *Locality) Reset() { *m = Locality{} } func (*Locality) ProtoMessage() {} func (*Locality) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{11} + return fileDescriptor_metadata_681e86602fa51f2e, []int{11} } func (m *Locality) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -697,7 +700,7 @@ type Tier struct { func (m *Tier) Reset() { *m = Tier{} } func (*Tier) ProtoMessage() {} func (*Tier) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{12} + return fileDescriptor_metadata_681e86602fa51f2e, []int{12} } func (m *Tier) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -736,7 +739,7 @@ type Version struct { func (m *Version) Reset() { *m = Version{} } func (*Version) ProtoMessage() {} func (*Version) Descriptor() ([]byte, []int) { - return fileDescriptor_metadata_4d08fb4df4010e8b, []int{13} + return fileDescriptor_metadata_681e86602fa51f2e, []int{13} } func (m *Version) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1253,6 +1256,14 @@ func (m *NodeDescriptor) MarshalTo(dAtA []byte) (int, error) { return 0, err } i += n9 + dAtA[i] = 0x5a + i++ + i = encodeVarintMetadata(dAtA, i, uint64(m.TenantAddress.Size())) + n10, err := m.TenantAddress.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n10 return i, nil } @@ -1274,19 +1285,19 @@ func (m *LocalityAddress) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintMetadata(dAtA, i, uint64(m.Address.Size())) - n10, err := m.Address.MarshalTo(dAtA[i:]) + n11, err := m.Address.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 dAtA[i] = 0x12 i++ i = encodeVarintMetadata(dAtA, i, uint64(m.LocalityTier.Size())) - n11, err := m.LocalityTier.MarshalTo(dAtA[i:]) + n12, err := m.LocalityTier.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n12 return i, nil } @@ -1311,27 +1322,27 @@ func (m *StoreDescriptor) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintMetadata(dAtA, i, uint64(m.Attrs.Size())) - n12, err := m.Attrs.MarshalTo(dAtA[i:]) + n13, err := m.Attrs.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n13 dAtA[i] = 0x1a i++ i = encodeVarintMetadata(dAtA, i, uint64(m.Node.Size())) - n13, err := m.Node.MarshalTo(dAtA[i:]) + n14, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 dAtA[i] = 0x22 i++ i = encodeVarintMetadata(dAtA, i, uint64(m.Capacity.Size())) - n14, err := m.Capacity.MarshalTo(dAtA[i:]) + n15, err := m.Capacity.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n15 return i, nil } @@ -1749,6 +1760,8 @@ func (m *NodeDescriptor) Size() (n int) { n += 1 + l + sovMetadata(uint64(l)) l = m.SQLAddress.Size() n += 1 + l + sovMetadata(uint64(l)) + l = m.TenantAddress.Size() + n += 1 + l + sovMetadata(uint64(l)) return n } @@ -3145,6 +3158,36 @@ func (m *NodeDescriptor) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TenantAddress", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.TenantAddress.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMetadata(dAtA[iNdEx:]) @@ -3955,95 +3998,96 @@ var ( ErrIntOverflowMetadata = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("roachpb/metadata.proto", fileDescriptor_metadata_4d08fb4df4010e8b) } +func init() { proto.RegisterFile("roachpb/metadata.proto", fileDescriptor_metadata_681e86602fa51f2e) } -var fileDescriptor_metadata_4d08fb4df4010e8b = []byte{ - // 1385 bytes of a gzipped FileDescriptorProto +var fileDescriptor_metadata_681e86602fa51f2e = []byte{ + // 1403 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0xcf, 0x6f, 0x1b, 0x45, - 0x14, 0xf6, 0xc6, 0xeb, 0x78, 0xfd, 0x9c, 0x1f, 0xf6, 0x08, 0x8a, 0x65, 0x84, 0x9d, 0x2e, 0xad, + 0x14, 0xf6, 0xc6, 0xeb, 0x78, 0xfd, 0x1c, 0x27, 0xf6, 0x08, 0x8a, 0x65, 0x84, 0x9d, 0x2e, 0xad, 0x68, 0x0b, 0x4a, 0xd2, 0xa0, 0xa8, 0x6a, 0xa0, 0x40, 0xdc, 0xa4, 0x95, 0xdb, 0xfc, 0x62, 0xe3, 0x16, 0x89, 0xcb, 0x6a, 0xb2, 0x3b, 0x38, 0x4b, 0xd7, 0xbb, 0xee, 0xec, 0x38, 0xad, 0xef, 0x48, - 0x20, 0x21, 0x24, 0x2e, 0x48, 0x1c, 0x2b, 0xa1, 0x9e, 0xb9, 0xf2, 0x27, 0xf4, 0xd8, 0x63, 0x4f, - 0x51, 0x49, 0x2f, 0x9c, 0x39, 0xf6, 0x80, 0xd0, 0xcc, 0xce, 0xec, 0xae, 0x53, 0x17, 0xda, 0x20, - 0x71, 0x9b, 0xfd, 0xde, 0xf7, 0xbd, 0x7d, 0xf3, 0xe6, 0xcd, 0x7b, 0x03, 0xa7, 0x68, 0x88, 0x9d, - 0xfd, 0xfe, 0xde, 0x42, 0x8f, 0x30, 0xec, 0x62, 0x86, 0xe7, 0xfb, 0x34, 0x64, 0x21, 0xaa, 0x3a, - 0xa1, 0x73, 0x47, 0xd8, 0xe6, 0x25, 0xa3, 0x5e, 0x1f, 0x30, 0xcf, 0x5f, 0x18, 0x04, 0x94, 0x44, - 0xa1, 0x7f, 0x40, 0x5c, 0x1b, 0xbb, 0x2e, 0x8d, 0xe9, 0xf5, 0x9a, 0xb0, 0xed, 0xfb, 0xce, 0x02, - 0xf3, 0x7a, 0x24, 0x62, 0xb8, 0xd7, 0x97, 0x96, 0x37, 0xba, 0x61, 0x37, 0x14, 0xcb, 0x05, 0xbe, - 0x8a, 0x51, 0xf3, 0x53, 0x80, 0x55, 0xc6, 0xa8, 0xb7, 0x37, 0x60, 0x24, 0x42, 0xef, 0x43, 0x01, - 0x33, 0x46, 0xa3, 0x9a, 0x36, 0x97, 0x3f, 0x57, 0x6a, 0xbd, 0xf9, 0xe7, 0x61, 0xb3, 0x3a, 0xc4, - 0x3d, 0x7f, 0xc5, 0x14, 0xf0, 0x07, 0x5f, 0xf9, 0xe1, 0x3d, 0xd3, 0x8a, 0x39, 0x2b, 0xfa, 0xcf, - 0x0f, 0x9a, 0x39, 0xf3, 0x5b, 0x0d, 0xaa, 0x16, 0xe9, 0xfb, 0x9e, 0x83, 0x99, 0x17, 0x06, 0x1d, - 0x4c, 0xbb, 0x84, 0xa1, 0x8b, 0x50, 0x0c, 0x42, 0x97, 0xd8, 0x9e, 0x5b, 0xd3, 0xe6, 0xb4, 0x73, - 0x85, 0x56, 0xed, 0xd1, 0x61, 0x33, 0x77, 0x74, 0xd8, 0x9c, 0xdc, 0x0a, 0x5d, 0xd2, 0x5e, 0x7b, - 0x9e, 0xac, 0xac, 0x49, 0x4e, 0x6c, 0xbb, 0x68, 0x19, 0x8c, 0x88, 0x85, 0x54, 0x68, 0x26, 0x84, - 0xa6, 0x2e, 0x35, 0xc5, 0x5d, 0x8e, 0x0b, 0x91, 0x5a, 0x5a, 0x45, 0xc1, 0x6d, 0xbb, 0x2b, 0x06, - 0x8f, 0xe2, 0x8f, 0x07, 0x4d, 0xcd, 0xfc, 0x2b, 0x8d, 0x64, 0x8d, 0x44, 0x0e, 0xf5, 0xfa, 0x2c, - 0xa4, 0xff, 0x5f, 0x24, 0xe8, 0x0a, 0x00, 0x8d, 0x7f, 0xcf, 0x85, 0x79, 0x21, 0x6c, 0x48, 0x61, - 0x49, 0x06, 0x26, 0xa4, 0xe9, 0x87, 0x55, 0x92, 0x8a, 0xb6, 0x8b, 0x96, 0x40, 0x67, 0xc3, 0x3e, - 0xa9, 0xe9, 0x73, 0xda, 0xb9, 0x99, 0xa5, 0xc6, 0xfc, 0x0b, 0xe7, 0x3e, 0x2f, 0x65, 0x9d, 0x61, - 0x9f, 0x58, 0x82, 0xbb, 0x32, 0xc5, 0x37, 0xff, 0xdb, 0x83, 0xa6, 0x26, 0x12, 0xf0, 0xbd, 0x06, - 0x53, 0xca, 0xb5, 0x4b, 0x02, 0xc6, 0x37, 0x42, 0x71, 0xd0, 0x4d, 0x36, 0x9f, 0x4f, 0x37, 0x62, - 0x71, 0x3c, 0xde, 0x88, 0x5c, 0x5a, 0x45, 0xc1, 0x6d, 0xbb, 0x68, 0x0d, 0x8a, 0x32, 0x2c, 0xb1, - 0xfd, 0xf2, 0xd2, 0x99, 0x97, 0x07, 0x93, 0x66, 0xba, 0xa5, 0x73, 0xdf, 0x96, 0x92, 0x9a, 0xbf, - 0xe6, 0x61, 0x56, 0xb8, 0xce, 0x1c, 0xc6, 0x09, 0x03, 0x3a, 0x0b, 0xa5, 0x88, 0x61, 0xca, 0xec, - 0x3b, 0x64, 0x28, 0x42, 0x9a, 0x6a, 0x19, 0xcf, 0x0f, 0x9b, 0xba, 0x75, 0x93, 0x0c, 0x2d, 0x43, - 0x98, 0x6e, 0x92, 0x21, 0x3a, 0x0d, 0x45, 0x12, 0xb8, 0x82, 0x94, 0x3f, 0x46, 0x9a, 0x24, 0x81, - 0xcb, 0x29, 0x5f, 0x40, 0xd5, 0x0b, 0x18, 0xa1, 0x01, 0xf6, 0x6d, 0x19, 0x68, 0x54, 0xd3, 0xe7, - 0xf2, 0xaf, 0xb9, 0xc9, 0x8a, 0x72, 0x22, 0x09, 0x11, 0xba, 0x01, 0xb3, 0x01, 0xb9, 0xcf, 0xec, - 0x4c, 0x05, 0x14, 0x44, 0x05, 0x98, 0x72, 0x83, 0xd3, 0x5b, 0xe4, 0x3e, 0x7b, 0x49, 0x15, 0x4c, - 0x07, 0x19, 0x9b, 0x8b, 0xce, 0x00, 0x74, 0x49, 0x40, 0xa8, 0xb8, 0x50, 0xb5, 0x49, 0x91, 0xa7, - 0xf8, 0xbf, 0x19, 0x1c, 0x7d, 0x0c, 0x10, 0x31, 0xcf, 0xb9, 0x33, 0xb4, 0xf7, 0x3c, 0x56, 0x2b, - 0x8a, 0x83, 0x7a, 0x27, 0xb3, 0x07, 0xde, 0x08, 0xe6, 0xf7, 0x7d, 0x67, 0xbe, 0xa3, 0x1a, 0x81, - 0x55, 0x8a, 0x05, 0x2d, 0x8f, 0x8d, 0x56, 0xce, 0x0d, 0xdd, 0x30, 0x2a, 0x25, 0xf3, 0xa1, 0x06, - 0xe5, 0x1d, 0x42, 0x1d, 0x12, 0x30, 0xcf, 0x27, 0x11, 0x3a, 0x05, 0xf9, 0xfe, 0xc5, 0x45, 0x71, - 0x50, 0x9a, 0x0c, 0x80, 0x03, 0x02, 0x5f, 0x5a, 0x16, 0x07, 0x91, 0xe2, 0x4b, 0xcb, 0x02, 0x5f, - 0x5e, 0x14, 0xb9, 0x4f, 0xf1, 0xe5, 0x98, 0x7f, 0x69, 0x59, 0x14, 0x76, 0x8a, 0x5f, 0x8a, 0xf9, - 0x97, 0x17, 0x45, 0x9e, 0x52, 0xfc, 0xf2, 0x22, 0xaa, 0x81, 0xde, 0xdf, 0xc4, 0xf7, 0xc5, 0xce, - 0x95, 0x41, 0x20, 0xb2, 0xe5, 0x3c, 0xcf, 0xc3, 0xb4, 0xb8, 0x7d, 0x57, 0x71, 0x1f, 0x3b, 0x1e, - 0x1b, 0xa2, 0x39, 0x30, 0x1c, 0xb9, 0x96, 0x75, 0x15, 0xab, 0x12, 0x14, 0x99, 0x50, 0xc2, 0x07, - 0xd8, 0xf3, 0xf1, 0x9e, 0x4f, 0x44, 0xe4, 0x8a, 0x92, 0xc2, 0xe8, 0x2c, 0x94, 0xe3, 0xea, 0x74, - 0xc2, 0x41, 0xc0, 0xe4, 0x0d, 0x96, 0x89, 0x17, 0x86, 0xab, 0x1c, 0xe7, 0x34, 0x9f, 0xe0, 0x48, - 0xd1, 0xf4, 0x2c, 0x4d, 0x18, 0x62, 0xda, 0x22, 0x54, 0xef, 0x51, 0x8f, 0x91, 0xc8, 0xee, 0x13, - 0x6a, 0x47, 0xc4, 0x09, 0x03, 0x77, 0x64, 0xaf, 0xb3, 0xb1, 0x79, 0x87, 0xd0, 0x5d, 0x61, 0x44, - 0x3b, 0x50, 0xdd, 0x1b, 0x2a, 0x81, 0xba, 0x81, 0x93, 0xe2, 0x60, 0xc7, 0xb5, 0x83, 0xcc, 0x51, - 0x29, 0x8f, 0x42, 0xbe, 0x43, 0xa8, 0xac, 0x25, 0x64, 0x01, 0xca, 0xc4, 0xa0, 0x5c, 0x16, 0x5f, - 0xc3, 0x65, 0x25, 0x09, 0x52, 0xf9, 0xac, 0x81, 0x3e, 0x88, 0x88, 0x5b, 0x33, 0x32, 0x49, 0x14, - 0x08, 0x3a, 0x0f, 0xd3, 0x7e, 0xd8, 0xf5, 0x1c, 0xec, 0xdb, 0x22, 0x90, 0x5a, 0x29, 0x43, 0x99, - 0x92, 0xa6, 0x16, 0xb7, 0xa0, 0x25, 0x40, 0x77, 0x07, 0x84, 0x7a, 0xa3, 0xd9, 0x81, 0x4c, 0x76, - 0x2a, 0xd2, 0x9e, 0xa4, 0x47, 0x1e, 0xfe, 0x53, 0x1d, 0x66, 0x78, 0xbf, 0xfe, 0x6f, 0x2d, 0xfe, - 0x13, 0x28, 0xf2, 0xa1, 0x49, 0xa2, 0x48, 0xb6, 0xb8, 0xc6, 0xf1, 0x9b, 0x73, 0x2b, 0x19, 0xaf, - 0xab, 0xae, 0x9b, 0x34, 0x37, 0x29, 0x42, 0x97, 0xd5, 0xa0, 0xcc, 0xbf, 0x70, 0xef, 0x54, 0x2e, - 0xd3, 0xb1, 0x2a, 0xc5, 0xb1, 0x02, 0x5d, 0x01, 0xc3, 0x0f, 0x1d, 0xec, 0xf3, 0x5a, 0xd5, 0x85, - 0xfa, 0xed, 0x31, 0xea, 0x0d, 0x49, 0x51, 0x85, 0xac, 0x24, 0xe8, 0x1a, 0x4c, 0xef, 0x12, 0x7a, - 0x40, 0xe8, 0x6d, 0x42, 0x23, 0xde, 0x1f, 0x0a, 0xc2, 0x47, 0x7d, 0x8c, 0x0f, 0xc9, 0x90, 0x2e, - 0x46, 0x65, 0xe8, 0x34, 0x94, 0xf6, 0x06, 0x9e, 0xef, 0xda, 0x0c, 0x77, 0x45, 0x91, 0x95, 0xd4, - 0xaf, 0x04, 0xdc, 0xc1, 0x5d, 0xf4, 0x2e, 0xef, 0x30, 0x98, 0x32, 0xfe, 0xc2, 0x88, 0x3b, 0x4c, - 0x72, 0x69, 0x24, 0xbe, 0xca, 0xd0, 0x2e, 0x54, 0x54, 0x6c, 0xb6, 0x4a, 0xa9, 0x21, 0x1a, 0xaa, - 0xf9, 0x0f, 0xdb, 0x5a, 0x8d, 0x99, 0xaa, 0x6e, 0xfd, 0x51, 0x18, 0xbd, 0x07, 0x53, 0x8e, 0x3f, - 0x88, 0x18, 0xa1, 0x76, 0x80, 0x7b, 0x44, 0x14, 0x92, 0x8a, 0xaf, 0x2c, 0x2d, 0x5b, 0xb8, 0x47, - 0xd0, 0x2e, 0x94, 0xa3, 0xbb, 0x7e, 0xf2, 0x63, 0x78, 0xa5, 0xb3, 0x44, 0xb2, 0x3c, 0x60, 0xf7, - 0xf3, 0x0d, 0xf9, 0x47, 0x0b, 0xa2, 0xbb, 0xbe, 0x5c, 0x9b, 0x3f, 0x69, 0x30, 0x7b, 0x2c, 0xd0, - 0x6c, 0xc1, 0x68, 0x27, 0x29, 0x98, 0x16, 0xbf, 0x1b, 0x32, 0x4d, 0xcc, 0x23, 0x54, 0x96, 0xdd, - 0x5b, 0x63, 0x72, 0xd4, 0xf1, 0x08, 0x4d, 0x2f, 0x4d, 0xac, 0xe1, 0x98, 0xf9, 0xcd, 0x04, 0xcc, - 0x8a, 0xbe, 0x37, 0x3a, 0x51, 0x93, 0xb7, 0x8a, 0xf6, 0xea, 0x6f, 0x95, 0xa4, 0x7e, 0x27, 0x5e, - 0xbb, 0x7e, 0x3f, 0x02, 0x9d, 0x5f, 0x22, 0x59, 0xf9, 0xa7, 0xc7, 0x28, 0x47, 0xaf, 0xa7, 0x6a, - 0x11, 0x5c, 0x84, 0x5a, 0x99, 0x46, 0x1d, 0x17, 0xff, 0xdc, 0x18, 0x07, 0x23, 0xcd, 0xfd, 0x78, - 0x2b, 0x37, 0x7f, 0xd0, 0xa0, 0x2a, 0xd3, 0x80, 0xdd, 0x64, 0x00, 0x9f, 0x30, 0x11, 0xab, 0x60, - 0x24, 0xef, 0x80, 0x09, 0x51, 0xb6, 0xcd, 0x97, 0xbf, 0x03, 0xc4, 0xab, 0x4a, 0xc5, 0xa3, 0x64, - 0xe6, 0x3a, 0x18, 0xaa, 0x5a, 0xd0, 0x87, 0x50, 0xe0, 0xa7, 0x1b, 0x3f, 0xa0, 0xff, 0xf5, 0x78, - 0x63, 0xae, 0x6c, 0x6c, 0x9f, 0x81, 0xce, 0x4d, 0x7c, 0x2a, 0xf2, 0x17, 0x8c, 0x96, 0x29, 0x79, - 0x0e, 0xa0, 0x3a, 0x14, 0x0e, 0xb0, 0x3f, 0x88, 0xa7, 0x97, 0xb2, 0xc4, 0x90, 0xf4, 0xf0, 0x50, - 0x83, 0xa2, 0xba, 0xde, 0x17, 0xa0, 0xd4, 0xc3, 0x5f, 0x87, 0xd4, 0x3e, 0xc0, 0xbe, 0xcc, 0xc7, - 0xb4, 0xcc, 0x47, 0x61, 0x93, 0x1b, 0x2c, 0x43, 0xd8, 0x6f, 0x63, 0x5f, 0x70, 0xbd, 0x40, 0x72, - 0x27, 0x8e, 0x71, 0xb9, 0xc1, 0x32, 0x84, 0x9d, 0x73, 0xeb, 0x50, 0xe8, 0x63, 0xe6, 0xec, 0x8f, - 0x4c, 0xc7, 0x18, 0xe2, 0x53, 0x78, 0x10, 0x44, 0x4c, 0x8c, 0xd8, 0xec, 0x54, 0x4c, 0xd0, 0x38, - 0xce, 0x0b, 0x3e, 0x94, 0x33, 0x4f, 0x59, 0x34, 0x03, 0x70, 0x7b, 0xbb, 0xb3, 0x6e, 0xd9, 0xd7, - 0x6e, 0x6d, 0x6c, 0x54, 0x72, 0x08, 0xc1, 0x4c, 0xfc, 0xdd, 0xde, 0xba, 0xba, 0xbd, 0xd9, 0xde, - 0xba, 0x5e, 0x99, 0x48, 0xb1, 0xed, 0x5b, 0x9d, 0xeb, 0xdb, 0x1c, 0xcb, 0xa7, 0xd8, 0xda, 0xfa, - 0xe6, 0x76, 0x87, 0x63, 0x3a, 0x2a, 0x43, 0x71, 0x63, 0x7d, 0xd5, 0xda, 0x5a, 0xb7, 0x2a, 0x5a, - 0x5d, 0xff, 0xee, 0x97, 0x46, 0xae, 0x75, 0xfe, 0xd1, 0xef, 0x8d, 0xdc, 0xa3, 0xa3, 0x86, 0xf6, - 0xf8, 0xa8, 0xa1, 0x3d, 0x39, 0x6a, 0x68, 0x4f, 0x8f, 0x1a, 0xda, 0x8f, 0xcf, 0x1a, 0xb9, 0xc7, - 0xcf, 0x1a, 0xb9, 0x27, 0xcf, 0x1a, 0xb9, 0x2f, 0x8b, 0xf2, 0x68, 0xfe, 0x0e, 0x00, 0x00, 0xff, - 0xff, 0x6e, 0xce, 0xc8, 0xc3, 0x84, 0x0d, 0x00, 0x00, + 0x20, 0x21, 0x24, 0x2e, 0x48, 0x1c, 0x2b, 0xa1, 0x1e, 0x11, 0x57, 0xfe, 0x84, 0x1e, 0x7b, 0xec, + 0x29, 0x82, 0xf4, 0xc2, 0x99, 0x63, 0x0f, 0x08, 0xcd, 0xec, 0xcc, 0xee, 0x3a, 0x4d, 0xa1, 0x29, + 0x12, 0x37, 0xfb, 0x7b, 0xdf, 0xf7, 0xf6, 0xcd, 0x9b, 0xf7, 0x63, 0xe0, 0x14, 0x0d, 0xb1, 0xb3, + 0x37, 0xd8, 0x9d, 0xef, 0x13, 0x86, 0x5d, 0xcc, 0xf0, 0xdc, 0x80, 0x86, 0x2c, 0x44, 0x35, 0x27, + 0x74, 0xee, 0x08, 0xdb, 0x9c, 0x64, 0x34, 0x1a, 0x43, 0xe6, 0xf9, 0xf3, 0xc3, 0x80, 0x92, 0x28, + 0xf4, 0xf7, 0x89, 0x6b, 0x63, 0xd7, 0xa5, 0x31, 0xbd, 0x51, 0x17, 0xb6, 0x3d, 0xdf, 0x99, 0x67, + 0x5e, 0x9f, 0x44, 0x0c, 0xf7, 0x07, 0xd2, 0xf2, 0x5a, 0x2f, 0xec, 0x85, 0xe2, 0xe7, 0x3c, 0xff, + 0x15, 0xa3, 0xe6, 0xc7, 0x00, 0x2b, 0x8c, 0x51, 0x6f, 0x77, 0xc8, 0x48, 0x84, 0xde, 0x85, 0x02, + 0x66, 0x8c, 0x46, 0x75, 0x6d, 0x36, 0x7f, 0xae, 0xd4, 0x7e, 0xfd, 0xcf, 0x83, 0x56, 0x6d, 0x84, + 0xfb, 0xfe, 0xb2, 0x29, 0xe0, 0xf7, 0xbe, 0xf0, 0xc3, 0x7b, 0xa6, 0x15, 0x73, 0x96, 0xf5, 0x1f, + 0x1f, 0xb4, 0x72, 0xe6, 0xd7, 0x1a, 0xd4, 0x2c, 0x32, 0xf0, 0x3d, 0x07, 0x33, 0x2f, 0x0c, 0xba, + 0x98, 0xf6, 0x08, 0x43, 0x17, 0xa1, 0x18, 0x84, 0x2e, 0xb1, 0x3d, 0xb7, 0xae, 0xcd, 0x6a, 0xe7, + 0x0a, 0xed, 0xfa, 0xa3, 0x83, 0x56, 0xee, 0xf0, 0xa0, 0x35, 0xb9, 0x19, 0xba, 0xa4, 0xb3, 0xfa, + 0x2c, 0xf9, 0x65, 0x4d, 0x72, 0x62, 0xc7, 0x45, 0x4b, 0x60, 0x44, 0x2c, 0xa4, 0x42, 0x33, 0x21, + 0x34, 0x0d, 0xa9, 0x29, 0xee, 0x70, 0x5c, 0x88, 0xd4, 0x4f, 0xab, 0x28, 0xb8, 0x1d, 0x77, 0xd9, + 0xe0, 0x51, 0xfc, 0xf1, 0xa0, 0xa5, 0x99, 0x7f, 0xa5, 0x91, 0xac, 0x92, 0xc8, 0xa1, 0xde, 0x80, + 0x85, 0xf4, 0xff, 0x8b, 0x04, 0x5d, 0x01, 0xa0, 0xf1, 0xe7, 0xb9, 0x30, 0x2f, 0x84, 0x4d, 0x29, + 0x2c, 0xc9, 0xc0, 0x84, 0x34, 0xfd, 0x63, 0x95, 0xa4, 0xa2, 0xe3, 0xa2, 0x45, 0xd0, 0xd9, 0x68, + 0x40, 0xea, 0xfa, 0xac, 0x76, 0x6e, 0x7a, 0xb1, 0x39, 0xf7, 0xdc, 0xbd, 0xcf, 0x49, 0x59, 0x77, + 0x34, 0x20, 0x96, 0xe0, 0x2e, 0x4f, 0xf1, 0xc3, 0xff, 0xfa, 0xa0, 0xa5, 0x89, 0x04, 0x7c, 0xab, + 0xc1, 0x94, 0x72, 0xed, 0x92, 0x80, 0xf1, 0x83, 0x50, 0x1c, 0xf4, 0x92, 0xc3, 0xe7, 0xd3, 0x83, + 0x58, 0x1c, 0x8f, 0x0f, 0x22, 0x7f, 0x5a, 0x45, 0xc1, 0xed, 0xb8, 0x68, 0x15, 0x8a, 0x32, 0x2c, + 0x71, 0xfc, 0xf2, 0xe2, 0x99, 0x17, 0x07, 0x93, 0x66, 0xba, 0xad, 0x73, 0xdf, 0x96, 0x92, 0x9a, + 0xbf, 0xe4, 0x61, 0x46, 0xb8, 0xce, 0x5c, 0xc6, 0x2b, 0x06, 0x74, 0x16, 0x4a, 0x11, 0xc3, 0x94, + 0xd9, 0x77, 0xc8, 0x48, 0x84, 0x34, 0xd5, 0x36, 0x9e, 0x1d, 0xb4, 0x74, 0xeb, 0x26, 0x19, 0x59, + 0x86, 0x30, 0xdd, 0x24, 0x23, 0x74, 0x1a, 0x8a, 0x24, 0x70, 0x05, 0x29, 0x7f, 0x84, 0x34, 0x49, + 0x02, 0x97, 0x53, 0x3e, 0x83, 0x9a, 0x17, 0x30, 0x42, 0x03, 0xec, 0xdb, 0x32, 0xd0, 0xa8, 0xae, + 0xcf, 0xe6, 0x4f, 0x78, 0xc8, 0xaa, 0x72, 0x22, 0x09, 0x11, 0xba, 0x01, 0x33, 0x01, 0xb9, 0xcf, + 0xec, 0x4c, 0x05, 0x14, 0x44, 0x05, 0x98, 0xf2, 0x80, 0x95, 0x4d, 0x72, 0x9f, 0xbd, 0xa0, 0x0a, + 0x2a, 0x41, 0xc6, 0xe6, 0xa2, 0x33, 0x00, 0x3d, 0x12, 0x10, 0x2a, 0x1a, 0xaa, 0x3e, 0x29, 0xf2, + 0x14, 0x7f, 0x37, 0x83, 0xa3, 0x0f, 0x01, 0x22, 0xe6, 0x39, 0x77, 0x46, 0xf6, 0xae, 0xc7, 0xea, + 0x45, 0x71, 0x51, 0x6f, 0x65, 0xce, 0xc0, 0x07, 0xc1, 0xdc, 0x9e, 0xef, 0xcc, 0x75, 0xd5, 0x20, + 0xb0, 0x4a, 0xb1, 0xa0, 0xed, 0xb1, 0xf1, 0xca, 0xb9, 0xa1, 0x1b, 0x46, 0xb5, 0x64, 0x3e, 0xd4, + 0xa0, 0xbc, 0x4d, 0xa8, 0x43, 0x02, 0xe6, 0xf9, 0x24, 0x42, 0xa7, 0x20, 0x3f, 0xb8, 0xb8, 0x20, + 0x2e, 0x4a, 0x93, 0x01, 0x70, 0x40, 0xe0, 0x8b, 0x4b, 0xe2, 0x22, 0x52, 0x7c, 0x71, 0x49, 0xe0, + 0x4b, 0x0b, 0x22, 0xf7, 0x29, 0xbe, 0x14, 0xf3, 0x2f, 0x2d, 0x89, 0xc2, 0x4e, 0xf1, 0x4b, 0x31, + 0xff, 0xf2, 0x82, 0xc8, 0x53, 0x8a, 0x5f, 0x5e, 0x40, 0x75, 0xd0, 0x07, 0x1b, 0xf8, 0xbe, 0x38, + 0xb9, 0x32, 0x08, 0x44, 0x8e, 0x9c, 0x67, 0x79, 0xa8, 0x88, 0xee, 0xbb, 0x8a, 0x07, 0xd8, 0xf1, + 0xd8, 0x08, 0xcd, 0x82, 0xe1, 0xc8, 0xdf, 0xb2, 0xae, 0x62, 0x55, 0x82, 0x22, 0x13, 0x4a, 0x78, + 0x1f, 0x7b, 0x3e, 0xde, 0xf5, 0x89, 0x88, 0x5c, 0x51, 0x52, 0x18, 0x9d, 0x85, 0x72, 0x5c, 0x9d, + 0x4e, 0x38, 0x0c, 0x98, 0xec, 0x60, 0x99, 0x78, 0x61, 0xb8, 0xca, 0x71, 0x4e, 0xf3, 0x09, 0x8e, + 0x14, 0x4d, 0xcf, 0xd2, 0x84, 0x21, 0xa6, 0x2d, 0x40, 0xed, 0x1e, 0xf5, 0x18, 0x89, 0xec, 0x01, + 0xa1, 0x76, 0x44, 0x9c, 0x30, 0x70, 0xc7, 0xce, 0x3a, 0x13, 0x9b, 0xb7, 0x09, 0xdd, 0x11, 0x46, + 0xb4, 0x0d, 0xb5, 0xdd, 0x91, 0x12, 0xa8, 0x0e, 0x9c, 0x14, 0x17, 0x7b, 0xdc, 0x38, 0xc8, 0x5c, + 0x95, 0xf2, 0x28, 0xe4, 0xdb, 0x84, 0xca, 0x5a, 0x42, 0x16, 0xa0, 0x4c, 0x0c, 0xca, 0x65, 0xf1, + 0x04, 0x2e, 0xab, 0x49, 0x90, 0xca, 0x67, 0x1d, 0xf4, 0x61, 0x44, 0xdc, 0xba, 0x91, 0x49, 0xa2, + 0x40, 0xd0, 0x79, 0xa8, 0xf8, 0x61, 0xcf, 0x73, 0xb0, 0x6f, 0x8b, 0x40, 0xea, 0xa5, 0x0c, 0x65, + 0x4a, 0x9a, 0xda, 0xdc, 0x82, 0x16, 0x01, 0xdd, 0x1d, 0x12, 0xea, 0x8d, 0x67, 0x07, 0x32, 0xd9, + 0xa9, 0x4a, 0x7b, 0x92, 0x1e, 0x79, 0xf9, 0x3f, 0x17, 0x60, 0x9a, 0xcf, 0xeb, 0xff, 0x36, 0xe2, + 0x3f, 0x82, 0x22, 0x5f, 0x9a, 0x24, 0x8a, 0xe4, 0x88, 0x6b, 0x1e, 0xed, 0x9c, 0x5b, 0xc9, 0x7a, + 0x5d, 0x71, 0xdd, 0x64, 0xb8, 0x49, 0x11, 0xba, 0xac, 0x16, 0x65, 0xfe, 0xb9, 0xbe, 0x53, 0xb9, + 0x4c, 0xd7, 0xaa, 0x14, 0xc7, 0x0a, 0x74, 0x05, 0x0c, 0x3f, 0x74, 0xb0, 0xcf, 0x6b, 0x55, 0x17, + 0xea, 0x37, 0x8f, 0x51, 0xaf, 0x4b, 0x8a, 0x2a, 0x64, 0x25, 0x41, 0xd7, 0xa0, 0xb2, 0x43, 0xe8, + 0x3e, 0xa1, 0xb7, 0x09, 0x8d, 0xf8, 0x7c, 0x28, 0x08, 0x1f, 0x8d, 0x63, 0x7c, 0x48, 0x86, 0x74, + 0x31, 0x2e, 0x43, 0xa7, 0xa1, 0xb4, 0x3b, 0xf4, 0x7c, 0xd7, 0x66, 0xb8, 0x27, 0x8a, 0xac, 0xa4, + 0x3e, 0x25, 0xe0, 0x2e, 0xee, 0xa1, 0xb7, 0xf9, 0x84, 0xc1, 0x94, 0xf1, 0x17, 0x46, 0x3c, 0x61, + 0x92, 0xa6, 0x91, 0xf8, 0x0a, 0x43, 0x3b, 0x50, 0x55, 0xb1, 0xd9, 0x2a, 0xa5, 0x86, 0x18, 0xa8, + 0xe6, 0x3f, 0x1c, 0x6b, 0x25, 0x66, 0xaa, 0xba, 0xf5, 0xc7, 0x61, 0xf4, 0x0e, 0x4c, 0x39, 0xfe, + 0x30, 0x62, 0x84, 0xda, 0x01, 0xee, 0x13, 0x51, 0x48, 0x2a, 0xbe, 0xb2, 0xb4, 0x6c, 0xe2, 0x3e, + 0x41, 0x3b, 0x50, 0x8e, 0xee, 0xfa, 0xc9, 0x87, 0xe1, 0xa5, 0xee, 0x12, 0xc9, 0xf2, 0x80, 0x9d, + 0x4f, 0xd7, 0xe5, 0x17, 0x2d, 0x88, 0xee, 0xfa, 0xea, 0xeb, 0x37, 0x61, 0x9a, 0x91, 0x00, 0x07, + 0x2c, 0xf1, 0x5b, 0x3e, 0x41, 0x8d, 0x54, 0x62, 0xad, 0x74, 0x66, 0xfe, 0xa0, 0xc1, 0xcc, 0x91, + 0x53, 0x67, 0xab, 0x4f, 0x7b, 0x95, 0xea, 0x6b, 0xf3, 0x46, 0x93, 0x39, 0x67, 0x1e, 0xa1, 0xb2, + 0x86, 0xdf, 0x38, 0x26, 0xe1, 0x5d, 0x8f, 0xd0, 0xb4, 0x03, 0x63, 0x0d, 0xc7, 0xcc, 0xaf, 0x26, + 0x60, 0x46, 0x0c, 0xd1, 0xf1, 0xf5, 0x9c, 0x3c, 0x7c, 0xb4, 0x97, 0x7f, 0xf8, 0x24, 0xcd, 0x30, + 0x71, 0xe2, 0x66, 0xf8, 0x00, 0x74, 0xde, 0x91, 0xb2, 0x8d, 0x4e, 0x1f, 0xa3, 0x1c, 0xef, 0x75, + 0x35, 0x6f, 0xb8, 0x08, 0xb5, 0x33, 0x53, 0x3f, 0xee, 0xa4, 0xd9, 0x63, 0x1c, 0x8c, 0x6d, 0x8a, + 0xa3, 0x7b, 0xc1, 0xfc, 0x4e, 0x83, 0x9a, 0x4c, 0x03, 0x76, 0x93, 0x6d, 0xfe, 0x8a, 0x89, 0x58, + 0x01, 0x23, 0x79, 0x54, 0x4c, 0x88, 0x1e, 0x68, 0xbd, 0xf8, 0x51, 0x21, 0x9e, 0x68, 0x2a, 0x1e, + 0x25, 0x33, 0xd7, 0xc0, 0x50, 0xd5, 0x82, 0xde, 0x87, 0x02, 0xbf, 0xdd, 0xf8, 0x35, 0xfe, 0xaf, + 0xd7, 0x1b, 0x73, 0xe5, 0x94, 0xfc, 0x04, 0x74, 0x6e, 0xe2, 0x2b, 0x96, 0x3f, 0x87, 0xb4, 0x4c, + 0xff, 0x70, 0x00, 0x35, 0xa0, 0xb0, 0x8f, 0xfd, 0x61, 0xbc, 0x0a, 0x95, 0x25, 0x86, 0xa4, 0x87, + 0x87, 0x1a, 0x14, 0xd5, 0xac, 0xb8, 0x00, 0xa5, 0x3e, 0xfe, 0x32, 0xa4, 0xf6, 0x3e, 0xf6, 0x65, + 0x3e, 0x2a, 0x32, 0x1f, 0x85, 0x0d, 0x6e, 0xb0, 0x0c, 0x61, 0xbf, 0x8d, 0x7d, 0xc1, 0xf5, 0x02, + 0xc9, 0x9d, 0x38, 0xc2, 0xe5, 0x06, 0xcb, 0x10, 0x76, 0xce, 0x6d, 0x40, 0x61, 0x80, 0x99, 0xb3, + 0x37, 0xb6, 0x6a, 0x63, 0x88, 0xaf, 0xf4, 0x61, 0x10, 0x31, 0xb1, 0xaf, 0xb3, 0x2b, 0x36, 0x41, + 0xe3, 0x38, 0x2f, 0xf8, 0x50, 0xce, 0xbc, 0x8b, 0xd1, 0x34, 0xc0, 0xed, 0xad, 0xee, 0x9a, 0x65, + 0x5f, 0xbb, 0xb5, 0xbe, 0x5e, 0xcd, 0x21, 0x04, 0xd3, 0xf1, 0xff, 0xce, 0xe6, 0xd5, 0xad, 0x8d, + 0xce, 0xe6, 0xf5, 0xea, 0x44, 0x8a, 0x6d, 0xdd, 0xea, 0x5e, 0xdf, 0xe2, 0x58, 0x3e, 0xc5, 0x56, + 0xd7, 0x36, 0xb6, 0xba, 0x1c, 0xd3, 0x51, 0x19, 0x8a, 0xeb, 0x6b, 0x2b, 0xd6, 0xe6, 0x9a, 0x55, + 0xd5, 0x1a, 0xfa, 0x37, 0x3f, 0x35, 0x73, 0xed, 0xf3, 0x8f, 0x7e, 0x6f, 0xe6, 0x1e, 0x1d, 0x36, + 0xb5, 0xc7, 0x87, 0x4d, 0xed, 0xc9, 0x61, 0x53, 0xfb, 0xed, 0xb0, 0xa9, 0x7d, 0xff, 0xb4, 0x99, + 0x7b, 0xfc, 0xb4, 0x99, 0x7b, 0xf2, 0xb4, 0x99, 0xfb, 0xbc, 0x28, 0xaf, 0xe6, 0xef, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xec, 0x92, 0x66, 0x18, 0xd1, 0x0d, 0x00, 0x00, } diff --git a/pkg/roachpb/metadata.proto b/pkg/roachpb/metadata.proto index 5dc565f98598..292d97fa4624 100644 --- a/pkg/roachpb/metadata.proto +++ b/pkg/roachpb/metadata.proto @@ -300,7 +300,11 @@ message NodeDescriptor { optional string cluster_name = 9 [(gogoproto.nullable) = false]; // The SQL address. If empty, indicates that the base address field // is also used to accept SQL connections. - optional util.UnresolvedAddr sql_address = 10 [(gogoproto.nullable) = false, (gogoproto.customname) = "SQLAddress"]; + optional util.UnresolvedAddr sql_address = 10 [(gogoproto.nullable) = false, + (gogoproto.customname) = "SQLAddress"]; + // The tenant KV address. If empty, indicates that the base address + // field is also used to accept tenant KV connections. + optional util.UnresolvedAddr tenant_address = 11 [(gogoproto.nullable) = false]; } // LocalityAddress holds the private address accessible only from other nodes diff --git a/pkg/rpc/tls.go b/pkg/rpc/tls.go index 5f4b50cee948..53862988b6b4 100644 --- a/pkg/rpc/tls.go +++ b/pkg/rpc/tls.go @@ -259,6 +259,9 @@ func (ctx *SecurityContext) CheckCertificateAddrs(cctx context.Context) { } } + // TODO(tbg): Verify that the tenant listen and advertise addresses are + // compatible with the provided certificate. + // Verify that the http listen and advertise addresses are // compatible with the provided certificate. certInfo = cm.UICert() diff --git a/pkg/server/drain.go b/pkg/server/drain.go index 3f886010202b..034fac8b286b 100644 --- a/pkg/server/drain.go +++ b/pkg/server/drain.go @@ -107,6 +107,9 @@ func (s *adminServer) Drain(req *serverpb.DrainRequest, stream serverpb.Admin_Dr // away (and who knows whether gRPC-goroutines are tied up in some // stopper task somewhere). s.server.grpc.Stop() + if s.server.tenantGRPC != nil { + s.server.tenantGRPC.Stop() + } s.server.stopper.Stop(ctx) }() @@ -204,6 +207,9 @@ func (s *Server) drainClients(ctx context.Context, reporter func(int, string)) e // Mark the server as draining in a way that probes to // /health?ready=1 will notice. s.grpc.setMode(modeDraining) + if s.tenantGRPC != nil { + s.tenantGRPC.setMode(modeDraining) + } // Wait for drainUnreadyWait. This will fail load balancer checks and // delay draining so that client traffic can move off this node. time.Sleep(drainWait.Get(&s.st.SV)) diff --git a/pkg/server/node.go b/pkg/server/node.go index c2b69cd8edde..289ac79a9ca3 100644 --- a/pkg/server/node.go +++ b/pkg/server/node.go @@ -333,7 +333,7 @@ func (n *Node) AnnotateCtxWithSpan( // NodeDescriptor is available, to help bootstrapping. func (n *Node) start( ctx context.Context, - addr, sqlAddr net.Addr, + addr, sqlAddr, tenantAddr net.Addr, state initState, clusterName string, attrs roachpb.Attributes, @@ -375,6 +375,7 @@ func (n *Node) start( NodeID: nodeID, Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), SQLAddress: util.MakeUnresolvedAddr(sqlAddr.Network(), sqlAddr.String()), + TenantAddress: util.MakeUnresolvedAddr(tenantAddr.Network(), tenantAddr.String()), Attrs: attrs, Locality: locality, LocalityAddress: localityAddress, diff --git a/pkg/server/server.go b/pkg/server/server.go index bddac7e95e42..2ae0c0aebfab 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -143,7 +143,9 @@ type Server struct { clock *hlc.Clock rpcContext *rpc.Context // The gRPC server on which the different RPC handlers will be registered. - grpc *grpcServer + grpc *grpcServer + // The optional tenant gRPC used by SQL tenants. + tenantGRPC *grpcServer gossip *gossip.Gossip nodeDialer *nodedialer.Dialer nodeLiveness *kvserver.NodeLiveness @@ -331,14 +333,20 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { // and after ValidateAddrs(). rpcContext.CheckCertificateAddrs(ctx) - grpcServer := newGRPCServer(rpcContext) + grpc := newGRPCServer(rpcContext) + + // TODO(tbg): pass a different rpcContext here. + var tenantGRPC *grpcServer + if cfg.SplitListenTenant { + tenantGRPC = newGRPCServer(rpcContext) + } g := gossip.New( cfg.AmbientCtx, &rpcContext.ClusterID, nodeIDContainer, rpcContext, - grpcServer.Server, + grpc.Server, stopper, registry, cfg.Locality, @@ -426,7 +434,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { ) raftTransport := kvserver.NewRaftTransport( - cfg.AmbientCtx, st, nodeDialer, grpcServer.Server, stopper, + cfg.AmbientCtx, st, nodeDialer, grpc.Server, stopper, ) tsDB := ts.NewDB(db, cfg.Settings) @@ -528,9 +536,12 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { storeCfg, recorder, registry, stopper, txnMetrics, nil /* execCfg */, &rpcContext.ClusterID) lateBoundNode = node - roachpb.RegisterInternalServer(grpcServer.Server, node) - kvserver.RegisterPerReplicaServer(grpcServer.Server, node.perReplicaServer) - node.storeCfg.ClosedTimestamp.RegisterClosedTimestampServer(grpcServer.Server) + roachpb.RegisterInternalServer(grpc.Server, node) + if tenantGRPC != nil { + roachpb.RegisterInternalServer(tenantGRPC.Server, node) + } + kvserver.RegisterPerReplicaServer(grpc.Server, node.perReplicaServer) + node.storeCfg.ClosedTimestamp.RegisterClosedTimestampServer(grpc.Server) replicationReporter := reports.NewReporter( db, node.stores, storePool, st, nodeLiveness, internalExecutor) @@ -573,7 +584,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { if reflect.ValueOf(gw).IsNil() { return nil, errors.Errorf("%d: nil", i) } - gw.RegisterService(grpcServer.Server) + gw.RegisterService(grpc.Server) } var jobAdoptionStopFile string @@ -592,7 +603,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { nodeLiveness: sqlbase.MakeOptionalNodeLiveness(nodeLiveness), gossip: gossip.MakeExposedGossip(g), nodeDialer: nodeDialer, - grpcServer: grpcServer.Server, + grpcServer: grpc.Server, recorder: recorder, nodeIDContainer: idContainer, externalStorage: externalStorage, @@ -625,7 +636,8 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { st: st, clock: clock, rpcContext: rpcContext, - grpc: grpcServer, + grpc: grpc, + tenantGRPC: tenantGRPC, gossip: g, nodeDialer: nodeDialer, nodeLiveness: nodeLiveness, @@ -764,21 +776,25 @@ func inspectEngines( // file", these are written once the listeners are available, before the server // is necessarily ready to serve. type listenerInfo struct { - listenRPC string // the (RPC) listen address, rewritten after name resolution and port allocation - advertiseRPC string // contains the original addr part of --listen/--advertise, with actual port number after port allocation if original was 0 - listenHTTP string // the HTTP endpoint - listenSQL string // the SQL endpoint, rewritten after name resolution and port allocation - advertiseSQL string // contains the original addr part of --sql-addr, with actual port number after port allocation if original was 0 + listenRPC string // the (RPC) listen address, rewritten after name resolution and port allocation + advertiseRPC string // contains the original addr part of --listen/--advertise, with actual port number after port allocation if original was 0 + listenSQL string // the SQL endpoint, rewritten after name resolution and port allocation + advertiseSQL string // contains the original addr part of --sql-addr, with actual port number after port allocation if original was 0 + listenTenant string // the tenant KV endpoint, rewritten after name resolution and port allocation + advertiseTenant string // contains the original addr part of --tenant-addr, with actual port number after port allocation if original was 0 + listenHTTP string // the HTTP endpoint } // Iter returns a mapping of file names to desired contents. func (li listenerInfo) Iter() map[string]string { return map[string]string{ - "cockroach.advertise-addr": li.advertiseRPC, - "cockroach.http-addr": li.listenHTTP, - "cockroach.listen-addr": li.listenRPC, - "cockroach.sql-addr": li.listenSQL, - "cockroach.advertise-sql-addr": li.advertiseSQL, + "cockroach.listen-addr": li.listenRPC, + "cockroach.advertise-addr": li.advertiseRPC, + "cockroach.sql-addr": li.listenSQL, + "cockroach.advertise-sql-addr": li.advertiseSQL, + "cockroach.tenant-addr": li.listenTenant, + "cockroach.advertise-tenant-addr": li.advertiseTenant, + "cockroach.http-addr": li.listenHTTP, } } @@ -1250,11 +1266,13 @@ func (s *Server) Start(ctx context.Context) error { // Write listener info files early in the startup sequence. `listenerInfo` has a comment. listenerFiles := listenerInfo{ - listenRPC: s.cfg.Addr, - advertiseRPC: s.cfg.AdvertiseAddr, - listenSQL: s.cfg.SQLAddr, - advertiseSQL: s.cfg.SQLAdvertiseAddr, - listenHTTP: s.cfg.HTTPAdvertiseAddr, + listenRPC: s.cfg.Addr, + advertiseRPC: s.cfg.AdvertiseAddr, + listenSQL: s.cfg.SQLAddr, + advertiseSQL: s.cfg.SQLAdvertiseAddr, + listenTenant: s.cfg.TenantAddr, + advertiseTenant: s.cfg.TenantAdvertiseAddr, + listenHTTP: s.cfg.HTTPAdvertiseAddr, }.Iter() for _, storeSpec := range s.cfg.Stores.Specs { @@ -1275,6 +1293,7 @@ func (s *Server) Start(ctx context.Context) error { listenAddrU := util.NewUnresolvedAddr("tcp", s.cfg.Addr) advAddrU := util.NewUnresolvedAddr("tcp", s.cfg.AdvertiseAddr) advSQLAddrU := util.NewUnresolvedAddr("tcp", s.cfg.SQLAdvertiseAddr) + advTenantAddrU := util.NewUnresolvedAddr("tcp", s.cfg.TenantAdvertiseAddr) filtered := s.cfg.FilterGossipBootstrapResolvers(ctx, listenAddrU, advAddrU) s.gossip.Start(advAddrU, filtered) @@ -1421,7 +1440,9 @@ func (s *Server) Start(ctx context.Context) error { // we're joining an existing cluster for the first time. if err := s.node.start( ctx, - advAddrU, advSQLAddrU, + advAddrU, + advSQLAddrU, + advTenantAddrU, *state, s.cfg.ClusterName, s.cfg.NodeAttributes, @@ -1457,7 +1478,15 @@ func (s *Server) Start(ctx context.Context) error { }) // We can now add the node registry. - s.recorder.AddNode(s.registry, s.node.Descriptor, s.node.startedAt, s.cfg.AdvertiseAddr, s.cfg.HTTPAdvertiseAddr, s.cfg.SQLAdvertiseAddr) + s.recorder.AddNode( + s.registry, + s.node.Descriptor, + s.node.startedAt, + s.cfg.AdvertiseAddr, + s.cfg.HTTPAdvertiseAddr, + s.cfg.SQLAdvertiseAddr, + s.cfg.TenantAdvertiseAddr, + ) // Begin recording runtime statistics. if err := s.startSampleEnvironment(ctx, base.DefaultMetricsSampleInterval); err != nil { @@ -1479,6 +1508,9 @@ func (s *Server) Start(ctx context.Context) error { }) s.grpc.setMode(modeOperational) + if s.tenantGRPC != nil { + s.tenantGRPC.setMode(modeOperational) + } log.Infof(ctx, "starting %s server at %s (use: %s)", s.cfg.HTTPRequestScheme(), s.cfg.HTTPAddr, s.cfg.HTTPAdvertiseAddr) @@ -1718,8 +1750,8 @@ func (s *Server) startListenRPCAndSQL( return pgwire.Match(r) }) // Also if the pg port is not split, the actual listen - // and advertise address for SQL become equal to that of RPC, - // regardless of what was configured. + // and advertise addresses for SQL become equal to that + // of RPC, regardless of what was configured. s.cfg.SQLAddr = s.cfg.Addr s.cfg.SQLAdvertiseAddr = s.cfg.AdvertiseAddr } @@ -1763,6 +1795,40 @@ func (s *Server) startListenRPCAndSQL( }) } + if s.cfg.SplitListenTenant { + tenantL, err := listen(ctx, &s.cfg.TenantAddr, &s.cfg.TenantAdvertiseAddr, "tenant") + if err != nil { + return nil, nil, err + } + log.Eventf(ctx, "listening on tenant port %s", s.cfg.TenantAddr) + + // The shutdown worker. + s.stopper.RunWorker(workersCtx, func(context.Context) { + <-s.stopper.ShouldQuiesce() + netutil.FatalIfUnexpected(tenantL.Close()) + <-s.stopper.ShouldStop() + s.tenantGRPC.Stop() + }) + + // Compose startup functions. + startPrimaryRPCServer := startRPCServer + startTenantRPCServer := func(ctx context.Context) { + s.stopper.RunWorker(workersCtx, func(context.Context) { + netutil.FatalIfUnexpected(s.tenantGRPC.Serve(tenantL)) + }) + } + startRPCServer = func(ctx context.Context) { + startPrimaryRPCServer(ctx) + startTenantRPCServer(ctx) + } + } else { + // If the tenant port is not split, the actual listen and advertise + // addresses for tenant KV become equal to that of RPC, regardless + // of what was configured. + s.cfg.TenantAddr = s.cfg.Addr + s.cfg.TenantAdvertiseAddr = s.cfg.AdvertiseAddr + } + return pgL, startRPCServer, nil } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 715d7ef481ee..05cacd3ef8a0 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -655,11 +655,13 @@ func TestListenerFileCreation(t *testing.T) { } li := listenerInfo{ - advertiseRPC: s.ServingRPCAddr(), - listenHTTP: s.HTTPAddr(), - listenRPC: s.RPCAddr(), - listenSQL: s.SQLAddr(), - advertiseSQL: s.ServingSQLAddr(), + listenRPC: s.RPCAddr(), + advertiseRPC: s.ServingRPCAddr(), + listenSQL: s.SQLAddr(), + advertiseSQL: s.ServingSQLAddr(), + listenTenant: s.TenantAddr(), + advertiseTenant: s.ServingTenantAddr(), + listenHTTP: s.HTTPAddr(), } expectedFiles := li.Iter() diff --git a/pkg/server/status/recorder.go b/pkg/server/status/recorder.go index 2e8388bb21c2..457771092e89 100644 --- a/pkg/server/status/recorder.go +++ b/pkg/server/status/recorder.go @@ -56,6 +56,7 @@ const ( advertiseAddrLabelKey = "advertise-addr" httpAddrLabelKey = "http-addr" sqlAddrLabelKey = "sql-addr" + tenantAddrLabelKey = "tenant-addr" ) type quantile struct { @@ -167,7 +168,7 @@ func (mr *MetricsRecorder) AddNode( reg *metric.Registry, desc roachpb.NodeDescriptor, startedAt int64, - advertiseAddr, httpAddr, sqlAddr string, + advertiseAddr, httpAddr, sqlAddr, tenantAddr string, ) { mr.mu.Lock() defer mr.mu.Unlock() @@ -186,6 +187,7 @@ func (mr *MetricsRecorder) AddNode( metadata.AddLabel(advertiseAddrLabelKey, advertiseAddr) metadata.AddLabel(httpAddrLabelKey, httpAddr) metadata.AddLabel(sqlAddrLabelKey, sqlAddr) + metadata.AddLabel(tenantAddrLabelKey, tenantAddr) nodeIDGauge := metric.NewGauge(metadata) nodeIDGauge.Update(int64(desc.NodeID)) reg.AddMetric(nodeIDGauge) diff --git a/pkg/server/status/recorder_test.go b/pkg/server/status/recorder_test.go index f4e6cf3f7d6a..ac3e472cf749 100644 --- a/pkg/server/status/recorder_test.go +++ b/pkg/server/status/recorder_test.go @@ -146,7 +146,7 @@ func TestMetricsRecorder(t *testing.T) { recorder := NewMetricsRecorder(hlc.NewClock(manual.UnixNano, time.Nanosecond), nil, nil, nil, st) recorder.AddStore(store1) recorder.AddStore(store2) - recorder.AddNode(reg1, nodeDesc, 50, "foo:26257", "foo:26258", "foo:5432") + recorder.AddNode(reg1, nodeDesc, 50, "foo:26257", "foo:26258", "foo:5432", "foo:26259") // Ensure the metric system's view of time does not advance during this test // as the test expects time to not advance too far which would age the actual diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index ed037df36760..4372a4137d7e 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -101,10 +101,13 @@ func makeTestBaseConfig(st *cluster.Settings) BaseConfig { // makeTestConfigFromParams). Call TestServer.ServingRPCAddr() and // .ServingSQLAddr() for the full address (including bound port). baseCfg.Addr = util.TestAddr.String() - baseCfg.SQLAddr = util.TestAddr.String() baseCfg.AdvertiseAddr = util.TestAddr.String() + baseCfg.SQLAddr = util.TestAddr.String() baseCfg.SQLAdvertiseAddr = util.TestAddr.String() baseCfg.SplitListenSQL = true + baseCfg.TenantAddr = util.TestAddr.String() + baseCfg.TenantAdvertiseAddr = util.TestAddr.String() + baseCfg.SplitListenTenant = true baseCfg.HTTPAddr = util.TestAddr.String() // Set standard user for intra-cluster traffic. baseCfg.User = security.NodeUser @@ -202,13 +205,9 @@ func makeTestConfigFromParams(params base.TestServerArgs) Config { cfg.AdvertiseAddr = util.IsolatedTestAddr.String() cfg.SQLAddr = util.IsolatedTestAddr.String() cfg.SQLAdvertiseAddr = util.IsolatedTestAddr.String() + cfg.TenantAddr = util.IsolatedTestAddr.String() + cfg.TenantAdvertiseAddr = util.IsolatedTestAddr.String() cfg.HTTPAddr = util.IsolatedTestAddr.String() - } else { - cfg.Addr = util.TestAddr.String() - cfg.AdvertiseAddr = util.TestAddr.String() - cfg.SQLAddr = util.TestAddr.String() - cfg.SQLAdvertiseAddr = util.TestAddr.String() - cfg.HTTPAddr = util.TestAddr.String() } if params.Addr != "" { cfg.Addr = params.Addr @@ -218,6 +217,10 @@ func makeTestConfigFromParams(params base.TestServerArgs) Config { cfg.SQLAddr = params.SQLAddr cfg.SQLAdvertiseAddr = params.SQLAddr } + if params.TenantAddr != "" { + cfg.TenantAddr = params.TenantAddr + cfg.TenantAdvertiseAddr = params.TenantAddr + } if params.HTTPAddr != "" { cfg.HTTPAddr = params.HTTPAddr } @@ -787,6 +790,12 @@ func (ts *TestServer) ServingSQLAddr() string { return ts.cfg.SQLAdvertiseAddr } +// ServingTenantAddr returns the server's Tenant address. Should be used by +// tenant SQL processes. +func (ts *TestServer) ServingTenantAddr() string { + return ts.cfg.TenantAdvertiseAddr +} + // HTTPAddr returns the server's HTTP address. Should be used by clients. func (ts *TestServer) HTTPAddr() string { return ts.cfg.HTTPAddr @@ -798,17 +807,23 @@ func (ts *TestServer) RPCAddr() string { return ts.cfg.Addr } -// DrainClients exports the drainClients() method for use by tests. -func (ts *TestServer) DrainClients(ctx context.Context) error { - return ts.drainClients(ctx, nil /* reporter */) -} - // SQLAddr returns the server's listening SQL address. // Note: use ServingSQLAddr() instead unless there is a specific reason not to. func (ts *TestServer) SQLAddr() string { return ts.cfg.SQLAddr } +// TenantAddr returns the server's listening Tenant address. +// Note: use ServingTenantAddr() instead unless there is a specific reason not to. +func (ts *TestServer) TenantAddr() string { + return ts.cfg.TenantAddr +} + +// DrainClients exports the drainClients() method for use by tests. +func (ts *TestServer) DrainClients(ctx context.Context) error { + return ts.drainClients(ctx, nil /* reporter */) +} + // WriteSummaries implements TestServerInterface. func (ts *TestServer) WriteSummaries() error { return ts.node.writeNodeStatus(context.TODO(), time.Hour) diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index 246c49140049..bdbf7279bb09 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -2689,11 +2689,7 @@ CREATE TABLE crdb_internal.gossip_nodes ( } listenAddrRPC := d.Address - listenAddrSQL := d.SQLAddress - if listenAddrSQL.IsEmpty() { - // Pre-19.2 node or same address for both. - listenAddrSQL = listenAddrRPC - } + listenAddrSQL := d.CheckedSQLAddress() advAddrRPC, err := g.GetNodeIDAddress(d.NodeID) if err != nil { diff --git a/pkg/testutils/serverutils/test_server_shim.go b/pkg/testutils/serverutils/test_server_shim.go index 4b16310c9fbc..cdfd1c8c2de3 100644 --- a/pkg/testutils/serverutils/test_server_shim.go +++ b/pkg/testutils/serverutils/test_server_shim.go @@ -56,6 +56,9 @@ type TestServerInterface interface { // ServingSQLAddr returns the server's advertised SQL address. ServingSQLAddr() string + // ServingTenantAddr returns the server's advertised Tenant address. + ServingTenantAddr() string + // HTTPAddr returns the server's http address. HTTPAddr() string @@ -67,6 +70,10 @@ type TestServerInterface interface { // Note: use ServingSQLAddr() instead unless specific reason not to. SQLAddr() string + // TenantAddr returns the server's Tenant address. + // Note: use ServingTenantAddr() instead unless specific reason not to. + TenantAddr() string + // DB returns a *client.DB instance for talking to this KV server. DB() *kv.DB From 3b604f6418e3cc239ab4f0d4b50310f601c2dc50 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Wed, 15 Jul 2020 13:03:54 -0400 Subject: [PATCH 2/4] sql: route tenant KV traffic to tenant KV address This commit hooks up the tenant proxy to route all KV traffic from a tenant SQL process to a node's tenant KV address, if on is configured. As of the previous commit, TestServer now listens to tenant KV traffic on a separate port (see `makeTestConfigFromParams`) and sets a TenantAddress field in its NodeDescriptor, so with this change, any logic test that uses `StartTenant` will now test this behavior. As a sanity check, if we don't set up a second gRPC server in `startListenRPCAndSQL`, all 3node-tenant logictests fail. --- pkg/kv/kvclient/kvcoord/node_store.go | 22 +--------------------- pkg/kv/kvclient/kvtenant/proxy.go | 16 ++++++++++++++++ pkg/roachpb/metadata.go | 3 --- pkg/server/testserver.go | 2 +- pkg/testutils/testcluster/testcluster.go | 1 - 5 files changed, 18 insertions(+), 26 deletions(-) diff --git a/pkg/kv/kvclient/kvcoord/node_store.go b/pkg/kv/kvclient/kvcoord/node_store.go index a2d8482122ba..d5024146e158 100644 --- a/pkg/kv/kvclient/kvcoord/node_store.go +++ b/pkg/kv/kvclient/kvcoord/node_store.go @@ -10,12 +10,7 @@ package kvcoord -import ( - "net" - - "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" -) +import "github.com/cockroachdb/cockroach/pkg/roachpb" // NodeDescStore stores a collection of NodeDescriptors. // @@ -25,18 +20,3 @@ type NodeDescStore interface { // It returns an error if the node is not known by the store. GetNodeDescriptor(roachpb.NodeID) (*roachpb.NodeDescriptor, error) } - -// AddressResolver wraps a NodeDescStore and a Locality and allows the pair to -// be used as a nodedialer.AddressResolver. -func AddressResolver(ns NodeDescStore, loc roachpb.Locality) nodedialer.AddressResolver { - return func(nodeID roachpb.NodeID) (net.Addr, error) { - nd, err := ns.GetNodeDescriptor(nodeID) - if err != nil { - return nil, err - } - return nd.AddressForLocality(loc), nil - } -} - -// Silence unused warning. -var _ = AddressResolver diff --git a/pkg/kv/kvclient/kvtenant/proxy.go b/pkg/kv/kvclient/kvtenant/proxy.go index e191ff9413b7..a82e1c22f9f0 100644 --- a/pkg/kv/kvclient/kvtenant/proxy.go +++ b/pkg/kv/kvclient/kvtenant/proxy.go @@ -14,9 +14,12 @@ package kvtenant import ( "context" + "net" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord" + "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" + "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/cockroachdb/errors" @@ -62,3 +65,16 @@ func (requiresCCLBinaryFactory) NewProxy( ) (Proxy, error) { return nil, errors.Errorf(`tenant proxy requires a CCL binary`) } + +// AddressResolver wraps a Proxy in an adapter that allows it be used as a +// nodedialer.AddressResolver. Addresses are resolved to a node's tenant KV +// address. See NodeDescriptor.CheckedTenantAddress. +func AddressResolver(p Proxy) nodedialer.AddressResolver { + return func(nodeID roachpb.NodeID) (net.Addr, error) { + nd, err := p.GetNodeDescriptor(nodeID) + if err != nil { + return nil, err + } + return nd.CheckedTenantAddress(), nil + } +} diff --git a/pkg/roachpb/metadata.go b/pkg/roachpb/metadata.go index 6846c355f70c..4c5cd699bac6 100644 --- a/pkg/roachpb/metadata.go +++ b/pkg/roachpb/metadata.go @@ -490,9 +490,6 @@ func (n *NodeDescriptor) CheckedTenantAddress() *util.UnresolvedAddr { return &n.TenantAddress } -// Silence unused warning. -var _ = (*NodeDescriptor).CheckedTenantAddress - // String returns a string representation of the Tier. func (t Tier) String() string { return fmt.Sprintf("%s=%s", t.Key, t.Value) diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index 4372a4137d7e..5bbcdf594eab 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -506,7 +506,7 @@ func makeSQLServerArgs( if err != nil { return sqlServerArgs{}, err } - resolver := kvcoord.AddressResolver(tenantProxy, baseCfg.Locality) + resolver := kvtenant.AddressResolver(tenantProxy) nodeDialer := nodedialer.New(rpcContext, resolver) dsCfg := kvcoord.DistSenderConfig{ diff --git a/pkg/testutils/testcluster/testcluster.go b/pkg/testutils/testcluster/testcluster.go index 1c5678b17f40..e4bbceac6290 100644 --- a/pkg/testutils/testcluster/testcluster.go +++ b/pkg/testutils/testcluster/testcluster.go @@ -199,7 +199,6 @@ func StartTestCluster(t testing.TB, nodes int, args base.TestClusterArgs) *TestC serverArgs.Knobs.Server.(*server.TestingKnobs).RPCListener = firstListener serverArgs.Addr = firstListener.Addr().String() } else { - //serverArgs.JoinAddr = tc.Servers[0].ServingRPCAddr() serverArgs.JoinAddr = firstListener.Addr().String() } From 14221e6263c818a2f91c7ed1511167cb71b31b15 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Wed, 15 Jul 2020 13:47:49 -0400 Subject: [PATCH 3/4] roachtest: configure --tenant-addr flag in acceptance/multitenant This commit configures the KV cluster in the `acceptance/multitenant` roachtest to use a separate tenant address, which instructs the nodes in the cluster to split off a second gRPC server on port 36257. --- pkg/cmd/roachtest/multitenant.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/roachtest/multitenant.go b/pkg/cmd/roachtest/multitenant.go index ba577c8909b2..c1edfa510fb4 100644 --- a/pkg/cmd/roachtest/multitenant.go +++ b/pkg/cmd/roachtest/multitenant.go @@ -22,13 +22,16 @@ import ( func runAcceptanceMultitenant(ctx context.Context, t *test, c *cluster) { c.Put(ctx, cockroach, "./cockroach") - c.Start(ctx, t, c.All()) + + // Start the cluster with tenant RPC servers running on different ports + // on each host. Don't bind to external interfaces when running locally. + tenantAddr := ifLocal("127.0.0.1", "0.0.0.0") + ":0" + c.Start(ctx, t, c.All(), startArgs("--args=--tenant-addr="+tenantAddr)) _, err := c.Conn(ctx, 1).Exec(`SELECT crdb_internal.create_tenant(123)`) require.NoError(t, err) - kvAddrs := c.ExternalAddr(ctx, c.All()) - + kvAddrs := tenantAddrs(ctx, t, c) errCh := make(chan error) go func() { errCh <- c.RunE(ctx, c.Node(1), @@ -73,3 +76,16 @@ func runAcceptanceMultitenant(ctx context.Context, t *test, c *cluster) { require.Equal(t, 1, id) require.Equal(t, "bar", v) } + +func tenantAddrs(ctx context.Context, t *test, c *cluster) (addrs []string) { + // NB: iterate to avoid needing to strip node ID from output lines. + for _, node := range c.All() { + const cmd = "cat {store-dir}/cockroach.advertise-tenant-addr" + out, err := c.RunWithBuffer(ctx, t.l, c.Node(node), cmd) + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + addrs = append(addrs, string(out)) + } + return addrs +} From eaa01fd678ba66a7836a07ae7377fe184ce6695d Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Thu, 16 Jul 2020 13:18:28 -0400 Subject: [PATCH 4/4] base: DRY flag names in Config.ValidateAddrs Removes some room for inconsistency. --- pkg/base/addr_validation.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pkg/base/addr_validation.go b/pkg/base/addr_validation.go index 8b7d8dafc10b..4f5817e813f7 100644 --- a/pkg/base/addr_validation.go +++ b/pkg/base/addr_validation.go @@ -17,6 +17,7 @@ import ( "os" "strconv" + "github.com/cockroachdb/cockroach/pkg/cli/cliflags" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" ) @@ -37,55 +38,55 @@ import ( func (cfg *Config) ValidateAddrs(ctx context.Context) error { // Validate the advertise address. advHost, advPort, err := validateAdvertiseAddr(ctx, - cfg.AdvertiseAddr, "--listen-addr", cfg.Addr, "") + cfg.AdvertiseAddr, cfg.Addr, "", cliflags.ListenAddr) if err != nil { - return errors.Wrap(err, "invalid --advertise-addr") + return invalidFlagErr(err, cliflags.AdvertiseAddr) } cfg.AdvertiseAddr = net.JoinHostPort(advHost, advPort) // Validate the RPC listen address. listenHost, listenPort, err := validateListenAddr(ctx, cfg.Addr, "") if err != nil { - return errors.Wrap(err, "invalid --listen-addr") + return invalidFlagErr(err, cliflags.ListenAddr) } cfg.Addr = net.JoinHostPort(listenHost, listenPort) // Validate the SQL advertise address. Use the provided advertise // addr as default. advSQLHost, advSQLPort, err := validateAdvertiseAddr(ctx, - cfg.SQLAdvertiseAddr, "--sql-addr", cfg.SQLAddr, advHost) + cfg.SQLAdvertiseAddr, cfg.SQLAddr, advHost, cliflags.ListenSQLAddr) if err != nil { - return errors.Wrap(err, "invalid --advertise-sql-addr") + return invalidFlagErr(err, cliflags.SQLAdvertiseAddr) } cfg.SQLAdvertiseAddr = net.JoinHostPort(advSQLHost, advSQLPort) // Validate the SQL listen address - use the resolved listen addr as default. sqlHost, sqlPort, err := validateListenAddr(ctx, cfg.SQLAddr, listenHost) if err != nil { - return errors.Wrap(err, "invalid --sql-addr") + return invalidFlagErr(err, cliflags.ListenSQLAddr) } cfg.SQLAddr = net.JoinHostPort(sqlHost, sqlPort) // Validate the tenant advertise address. Use the provided advertise // addr as default. advTenantHost, advTenantPort, err := validateAdvertiseAddr(ctx, - cfg.TenantAdvertiseAddr, "--tenant-addr", cfg.TenantAddr, advHost) + cfg.TenantAdvertiseAddr, cfg.TenantAddr, advHost, cliflags.ListenTenantAddr) if err != nil { - return errors.Wrap(err, "invalid --advertise-tenant-addr") + return invalidFlagErr(err, cliflags.TenantAdvertiseAddr) } cfg.TenantAdvertiseAddr = net.JoinHostPort(advTenantHost, advTenantPort) // Validate the tenant listen address - use the resolved listen addr as default. tenantHost, tenantPort, err := validateListenAddr(ctx, cfg.TenantAddr, listenHost) if err != nil { - return errors.Wrap(err, "invalid --tenant-addr") + return invalidFlagErr(err, cliflags.ListenTenantAddr) } cfg.TenantAddr = net.JoinHostPort(tenantHost, tenantPort) // Validate the HTTP advertise address. Use the provided advertise // addr as default. advHTTPHost, advHTTPPort, err := validateAdvertiseAddr(ctx, - cfg.HTTPAdvertiseAddr, "--http-addr", cfg.HTTPAddr, advHost) + cfg.HTTPAdvertiseAddr, cfg.HTTPAddr, advHost, cliflags.ListenHTTPAddr) if err != nil { return errors.Wrap(err, "cannot compute public HTTP address") } @@ -95,7 +96,7 @@ func (cfg *Config) ValidateAddrs(ctx context.Context) error { // as default. httpHost, httpPort, err := validateListenAddr(ctx, cfg.HTTPAddr, listenHost) if err != nil { - return errors.Wrap(err, "invalid --http-addr") + return invalidFlagErr(err, cliflags.ListenHTTPAddr) } cfg.HTTPAddr = net.JoinHostPort(httpHost, httpPort) return nil @@ -142,16 +143,16 @@ func UpdateAddrs(ctx context.Context, addr, advAddr *string, ln net.Addr) error return nil } -// validateAdvertiseAddr validates an normalizes an address suitable +// validateAdvertiseAddr validates and normalizes an address suitable // for use in gossiping - for use by other nodes. This ensures // that if the "host" part is empty, it gets filled in with // the configured listen address if any, or the canonical host name. func validateAdvertiseAddr( - ctx context.Context, advAddr, flag, listenAddr, defaultHost string, + ctx context.Context, advAddr, listenAddr, defaultHost string, listenFlag cliflags.FlagInfo, ) (string, string, error) { listenHost, listenPort, err := getListenAddr(listenAddr, defaultHost) if err != nil { - return "", "", errors.Wrapf(err, "invalid %s", flag) + return "", "", invalidFlagErr(err, listenFlag) } advHost, advPort := "", "" @@ -280,3 +281,7 @@ func LookupAddr(ctx context.Context, resolver *net.Resolver, host string) (strin // No IPv4 address, return the first resolved address instead. return addrs[0].String(), nil } + +func invalidFlagErr(err error, flag cliflags.FlagInfo) error { + return errors.Wrapf(err, "invalid --%s", flag.Name) +}