From 25fb0ae676c1805b75674cb020abf50219db3018 Mon Sep 17 00:00:00 2001 From: Matthew B White Date: Wed, 10 Nov 2021 10:55:25 +0000 Subject: [PATCH] Setup basic HA features for the Peers - Add a new label to each peer to mark which org it is in - Create new service per org, that matches this label - so the service can pick from one of multiple pods - Update the kubeproxy to give more choice of ha stratergies - Update the application configmaps and samples to refer to this new service rather than specific peers Signed-off-by: Matthew B White --- test-network-k8s/docs/HIGH_AVAILABILITY.md | 71 +++++++++++++++++++ test-network-k8s/kube/org1/org1-peer1.yaml | 15 +++- test-network-k8s/kube/org1/org1-peer2.yaml | 1 + test-network-k8s/kube/org2/org2-peer1.yaml | 15 +++- test-network-k8s/kube/org2/org2-peer2.yaml | 1 + .../scripts/application_connection.sh | 4 +- test-network-k8s/scripts/chaincode.sh | 17 +++++ test-network-k8s/scripts/kind.sh | 4 ++ test-network-k8s/scripts/test_network.sh | 8 +-- 9 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 test-network-k8s/docs/HIGH_AVAILABILITY.md diff --git a/test-network-k8s/docs/HIGH_AVAILABILITY.md b/test-network-k8s/docs/HIGH_AVAILABILITY.md new file mode 100644 index 0000000000..d23a411a48 --- /dev/null +++ b/test-network-k8s/docs/HIGH_AVAILABILITY.md @@ -0,0 +1,71 @@ +# High Availability + +The peers have been configured so they implemented a essential failover/high-availability configuration. + +Two important notes: + +1. The word 'gateway' in the k8s definitions is being used in a generic way. It is not tied to the concept of the 'Fabric Gateway' component. However using the 'Fabric-Gateway' with the udpated SDKs, make connecting to Fabric even easier. There is a single connection, that can easily be handled with core k8s abilities. Attempting the approach described below with the older SDKs is not recommended. +2. Long Lived gRPC connections. Remember that the connections between components in Fabric are long-lived gRPC connections. From a client application's perspective that means the connection will be load-balanced when initially connected, but unless the connection breaks, it will not be 're-load-balanced'. It's important to keep this in mind. + +## Peer Gateway Services + +Each peer has defined their own K8S service, with the selector specifically choosing only one peer pod. +In this test-network, there are two peers per organization. Using a service with a different selector that +picks both peer pods, allows a degree of load balancing. + +```yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: org2-peer-gateway-svc +spec: + ports: + - name: gossip + port: 7051 + protocol: TCP + selector: + org: org2 +``` + +The selector is `org: org2` that is defined in the specification of the Peer's Deployment. + +```yaml + template: + metadata: + labels: + app: org2-peer1 + org: org2 +``` + +## Kube Proxy Configuration +The proxy configuration is set to be `ipvs`. This gives a lot more scope for different load balancing algorthms. +"Round Robin" is the default configuration (as used in this test network). For more information check this [deep dive](https://kubernetes.io/blog/2018/07/09/ipvs-based-in-cluster-load-balancing-deep-dive) on the Kubernetes blog. + +For this KIND cluster, this is configured by updating the cluster configuration, add the following yaml. + +```yaml +networking: + kubeProxyMode: "ipvs" +``` + +## Application and TLS Configuration + +It is important that applications connect to the `org2-peer-gateway-svc` or `org1-peer-gateway-svc` rather that specific peer services. That way the service can load balance. However if TLS used, errors will occur as the host name that is connected to is different to that used by the application. + +The solution is to add the additional servicename to the hosts field in the SAN section of the TLS certificate. As an example here is the command that is used to create the TLS certificate for org1-peer1. Note the + +```bash +fabric-ca-client enroll --url https://org1-peer1:peerpw@org1-ecert-ca --csr.hosts org1-peer1,org1-peer-gateway-svc --mspdir /var/hyperledger/fabric/organizations/peerOrganizations/org1.example.com/peers/org1-peer1.org1.example.com/msp +``` + +## Summary + +The FabricGateway and updated SDKs, improve the connection from a client application to Fabric, by needing only a single connection to one peer. By using a K8S service fronting two or more peer pods, a degree of load-balancing can be achieved. Remember that this will only be load balanced when the connection is first created. If a single peer becomes heavily loaded, K8S will not move any existing connection. + +To achieve this you would need to have a monitoring system that can trigger applications to disconnect and reconnect. + +If the connection drops, the application can reconnect and will get to a working peer. + + + diff --git a/test-network-k8s/kube/org1/org1-peer1.yaml b/test-network-k8s/kube/org1/org1-peer1.yaml index cb879f026e..f1f40ac43b 100644 --- a/test-network-k8s/kube/org1/org1-peer1.yaml +++ b/test-network-k8s/kube/org1/org1-peer1.yaml @@ -43,6 +43,7 @@ spec: metadata: labels: app: org1-peer1 + org: org1 spec: containers: - name: main @@ -100,4 +101,16 @@ spec: port: 9443 protocol: TCP selector: - app: org1-peer1 \ No newline at end of file + app: org1-peer1 +--- +apiVersion: v1 +kind: Service +metadata: + name: org1-peer-gateway-svc +spec: + ports: + - name: gossip + port: 7051 + protocol: TCP + selector: + org: org1 \ No newline at end of file diff --git a/test-network-k8s/kube/org1/org1-peer2.yaml b/test-network-k8s/kube/org1/org1-peer2.yaml index a0e994265a..78f57862a5 100644 --- a/test-network-k8s/kube/org1/org1-peer2.yaml +++ b/test-network-k8s/kube/org1/org1-peer2.yaml @@ -43,6 +43,7 @@ spec: metadata: labels: app: org1-peer2 + org: org1 spec: containers: - name: main diff --git a/test-network-k8s/kube/org2/org2-peer1.yaml b/test-network-k8s/kube/org2/org2-peer1.yaml index b77bd70d78..7dba924da2 100644 --- a/test-network-k8s/kube/org2/org2-peer1.yaml +++ b/test-network-k8s/kube/org2/org2-peer1.yaml @@ -43,6 +43,7 @@ spec: metadata: labels: app: org2-peer1 + org: org2 spec: containers: - name: main @@ -101,4 +102,16 @@ spec: port: 9443 protocol: TCP selector: - app: org2-peer1 \ No newline at end of file + app: org2-peer1 +--- +apiVersion: v1 +kind: Service +metadata: + name: org2-peer-gateway-svc +spec: + ports: + - name: gossip + port: 7051 + protocol: TCP + selector: + org: org2 diff --git a/test-network-k8s/kube/org2/org2-peer2.yaml b/test-network-k8s/kube/org2/org2-peer2.yaml index e68363c98f..3b80025fb3 100644 --- a/test-network-k8s/kube/org2/org2-peer2.yaml +++ b/test-network-k8s/kube/org2/org2-peer2.yaml @@ -43,6 +43,7 @@ spec: metadata: labels: app: org2-peer2 + org: org2 spec: containers: - name: main diff --git a/test-network-k8s/scripts/application_connection.sh b/test-network-k8s/scripts/application_connection.sh index 9563b12850..b6b67c7bf7 100755 --- a/test-network-k8s/scripts/application_connection.sh +++ b/test-network-k8s/scripts/application_connection.sh @@ -100,8 +100,8 @@ data: fabric_channel: ${CHANNEL_NAME} fabric_contract: ${CHAINCODE_NAME} fabric_wallet_dir: /fabric/application/wallet - fabric_gateway_hostport: org1-peer1:7051 - fabric_gateway_sslHostOverride: org1-peer1 + fabric_gateway_hostport: org1-peer-svc:7051 + fabric_gateway_sslHostOverride: org1-peer-svc fabric_user: appuser_org1 fabric_gateway_tlsCertPath: /fabric/tlscacerts/org1-tls-ca.pem EOF diff --git a/test-network-k8s/scripts/chaincode.sh b/test-network-k8s/scripts/chaincode.sh index c0fec2210c..cb06618863 100755 --- a/test-network-k8s/scripts/chaincode.sh +++ b/test-network-k8s/scripts/chaincode.sh @@ -111,10 +111,27 @@ function query_chaincode_metadata() { set -x local args='{"Args":["org.hyperledger.fabric:GetMetadata"]}' # todo: mangle additional $@ parameters with bash escape quotations + log 'Org1-Peer1:' echo ' export CORE_PEER_ADDRESS=org1-peer1:7051 peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"' ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + + log '' + log 'Org1-Peer2:' + echo ' + export CORE_PEER_ADDRESS=org1-peer2:7051 + peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"' + ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + + log '' + log 'Org1-Peer-SVC:' + echo ' + export CORE_PEER_ADDRESS=org1-peer-svc:7051 + peer chaincode query -n '${CHAINCODE_NAME}' -C '${CHANNEL_NAME}' -c '"'$args'"' + ' | exec kubectl -n $NS exec deploy/org1-admin-cli -c main -i -- /bin/bash + + } function invoke_chaincode() { diff --git a/test-network-k8s/scripts/kind.sh b/test-network-k8s/scripts/kind.sh index a6fb1fd6f2..67a3dd6ab2 100755 --- a/test-network-k8s/scripts/kind.sh +++ b/test-network-k8s/scripts/kind.sh @@ -54,6 +54,8 @@ function kind_create() { local ingress_http_port=${NGINX_HTTP_PORT} local ingress_https_port=${NGINX_HTTPS_PORT} + # the 'ipvs'proxy mode permits better HA abilities + cat <