FAB-9025 Mutual auth for integration tests

Change-Id: I29052d167a466bf63d49e83a9db46e11f2c35342
Signed-off-by: rickr <cr22rc@gmail.com>

src/main/java/org/hyperledger/fabric/sdk/NetworkConfig.java

@@ -130,7 +130,7 @@ private Properties getNodeProperties(String type, String name, Map<String, Node>
             return new Properties();
         } else {
 
-            return new Properties(node.properties);
+            return (Properties) node.properties.clone();
         }
 
     }

src/main/java/org/hyperledger/fabric/sdk/OrdererClient.java

@@ -23,6 +23,8 @@
 import io.grpc.ConnectivityState;
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
 import io.grpc.stub.StreamObserver;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -180,6 +182,12 @@ public void onCompleted() {
                     throw ste;
                 }
                 if (throwable[0] != null) {
+                    Throwable t = throwable[0];
+                    if (t instanceof StatusRuntimeException) {
+                        StatusRuntimeException sre = (StatusRuntimeException) t;
+                        Status status = sre.getStatus();
+                        logger.error(format("grpc status Code:%s, Description %s, ", status.getDescription(), status.getCode() + ""), sre.getCause());
+                    }
                     //get full stack trace
                     TransactionException ste = new TransactionException(format("Channel %s, send transaction failed on orderer %s. Reason: %s",
                             channelName, name, throwable[0].getMessage()), throwable[0]);

src/test/cirun.sh

@@ -12,6 +12,7 @@
 # unset ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CA_TLS
 export ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_TLS=true
 export ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CA_TLS=--tls.enabled
+export ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CLIENT_AUTH_REQUIRED=true
 
 export ORG_HYPERLEDGER_FABRIC_SDK_LOGLEVEL=TRACE
 export ORG_HYPERLEDGER_FABRIC_CA_SDK_LOGLEVEL=TRACE
@@ -34,6 +35,7 @@ ORG_HYPERLEDGER_FABRIC_SDKTEST_VERSION=${ORG_HYPERLEDGER_FABRIC_SDKTEST_VERSION:
 
 if [ "$ORG_HYPERLEDGER_FABRIC_SDKTEST_VERSION" == "1.0.0" ]; then
 # Limit the test run for V1.0
+export ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CLIENT_AUTH_REQUIRED=false
 export ORG_HYPERLEDGER_FABRIC_SDKTEST_ITSUITE="-Dorg.hyperledger.fabric.sdktest.ITSuite=IntegrationSuiteV1.java"
 #Options starting fabric-ca in docker-compose.yaml which are not supported on v1.0
 export V11_IDENTITIES_ALLOWREMOVE=""

src/test/fixture/sdkintegration/.env

@@ -9,6 +9,7 @@
 #ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CA_TLS=--tls.enabled
 ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_TLS=false
 ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CA_TLS=
+ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CLIENT_AUTH_REQUIRED=false
 #
 #Image tags:
 #IMAGE_TAG_FABRIC=:x86_64-1.1.0-alpha

src/test/fixture/sdkintegration/docker-compose.yaml

@@ -41,9 +41,11 @@ services:
       - ORDERER_GENERAL_LOCALMSPID=OrdererMSP
       - ORDERER_GENERAL_LOCALMSPDIR=/etc/hyperledger/msp/orderer/msp
       - ORDERER_GENERAL_TLS_ENABLED=${ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_TLS}
+      - ORDERER_GENERAL_TLS_CLIENTAUTHREQUIRED=${ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CLIENT_AUTH_REQUIRED}
       - ORDERER_GENERAL_TLS_PRIVATEKEY=/etc/hyperledger/msp/orderer/tls/server.key
       - ORDERER_GENERAL_TLS_CERTIFICATE=/etc/hyperledger/msp/orderer/tls/server.crt
       - ORDERER_GENERAL_TLS_ROOTCAS=[/etc/hyperledger/msp/orderer/tls/ca.crt]
+      - ORDERER_GENERAL_TLS_CLIENTROOTCAS=[/etc/hyperledger/msp/peerOrg1/msp/tlscacerts/tlsca.org1.example.com-cert.pem, /etc/hyperledger/msp/peerOrg2/msp/tlscacerts/tlsca.org2.example.com-cert.pem]
       - GRPC_TRACE=all=true,
       - GRPC_VERBOSITY=debug
       - ORDERER_GENERAL_AUTHENTICATION_TIMEWINDOW=3600s #Not for production -- remove.
@@ -52,6 +54,9 @@ services:
     volumes:
      - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}:/etc/hyperledger/configtx:ro
      - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/ordererOrganizations/example.com/orderers/orderer.example.com/:/etc/hyperledger/msp/orderer:ro
+     - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/:/etc/hyperledger/msp/peerOrg1:ro
+     - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/:/etc/hyperledger/msp/peerOrg2:ro
+
     ports:
       - 7050:7050
 
@@ -66,6 +71,9 @@ services:
       - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer0.org1.example.com:7051
 #      - CORE_PEER_GOSSIP_ORGLEADER=true
       - CORE_PEER_LOCALMSPID=Org1MSP
+
+#      - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/msp/peer/msp/cacerts/ca.org1.example.com-cert.pem
+      - CORE_PEER_TLS_CLIENTROOTCAS_FILES=/etc/hyperledger/msp/peer/msp/tlscacerts/tlsca.org1.example.com-cert.pem
     volumes:
       - /var/run/:/host/var/run/
       - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/:/etc/hyperledger/msp/peer:ro
@@ -86,6 +94,8 @@ services:
       - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer1.org1.example.com:7051
 #      - CORE_PEER_GOSSIP_BOOTSTRAP=peer0.org1.example.com:7051
       - CORE_PEER_LOCALMSPID=Org1MSP
+#      - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/msp/peer/msp/cacerts/ca.org1.example.com-cert.pem
+      - CORE_PEER_TLS_CLIENTROOTCAS_FILES=/etc/hyperledger/msp/peer/msp/tlscacerts/tlsca.org1.example.com-cert.pem
     volumes:
          - /var/run/:/host/var/run/
          - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org1.example.com/peers/peer1.org1.example.com/:/etc/hyperledger/msp/peer:ro
@@ -107,6 +117,8 @@ services:
       - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer0.org2.example.com:8051
 #      - CORE_PEER_GOSSIP_ORGLEADER=true
       - CORE_PEER_LOCALMSPID=Org2MSP
+#      - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/msp/peer/msp/cacerts/ca.org2.example.com-cert.pem
+      - CORE_PEER_TLS_CLIENTROOTCAS_FILES=/etc/hyperledger/msp/peer/msp/tlscacerts/tlsca.org2.example.com-cert.pem
     volumes:
       - /var/run/:/host/var/run/
       - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/:/etc/hyperledger/msp/peer:ro
@@ -127,6 +139,8 @@ services:
       - CORE_PEER_GOSSIP_EXTERNALENDPOINT=peer1.org2.example.com:8051
 #      - CORE_PEER_GOSSIP_BOOTSTRAP=peer0.org2.example.com:8051
       - CORE_PEER_LOCALMSPID=Org2MSP
+#      - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/msp/peer/msp/cacerts/ca.org2.example.com-cert.pem
+      - CORE_PEER_TLS_CLIENTROOTCAS_FILES=/etc/hyperledger/msp/peer/msp/tlscacerts/tlsca.org2.example.com-cert.pem
     volumes:
          - /var/run/:/host/var/run/
          - ./e2e-2Orgs/${FAB_CONFIG_GEN_VERS}/crypto-config/peerOrganizations/org2.example.com/peers/peer1.org2.example.com/:/etc/hyperledger/msp/peer:ro

src/test/fixture/sdkintegration/peer-base/peer-base.yaml

@@ -19,10 +19,12 @@ services:
       - CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/msp/peer/msp
       - CORE_PEER_GOSSIP_SKIPHANDSHAKE=true
       - CORE_PEER_PROFILE_ENABLED=false
+      - CORE_PEER_TLS_ENABLED=${ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_TLS}
+      - CORE_PEER_TLS_CLIENTAUTHREQUIRED=${ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_CLIENT_AUTH_REQUIRED}
       - CORE_PEER_TLS_CERT_FILE=/etc/hyperledger/msp/peer/tls/server.crt
       - CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/msp/peer/tls/server.key
       - CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/msp/peer/tls/ca.crt
-      - CORE_PEER_TLS_ENABLED=${ORG_HYPERLEDGER_FABRIC_SDKTEST_INTEGRATIONTESTS_TLS}
+
       # NOTE BOTH below should never be used in production.
       - CORE_PEER_AUTHENTICATION_TIMEWINDOW=3600s
       - CORE_PEER_EVENTS_TIMEWINDOW=3600s

src/test/java/org/hyperledger/fabric/sdk/testutils/TestConfig.java

@@ -295,7 +295,8 @@ public Properties getOrdererProperties(String name) {
 
     }
 
-    private Properties getEndPointProperties(final String type, final String name) {
+    public Properties getEndPointProperties(final String type, final String name) {
+        Properties ret = new Properties();
 
         final String domainName = getDomainName(name);
 
@@ -306,9 +307,33 @@ private Properties getEndPointProperties(final String type, final String name) {
                     cert.getAbsolutePath()));
         }
 
-        Properties ret = new Properties();
+        if (!isRunningAgainstFabric10()) {
+            File clientCert;
+            File clientKey;
+            if ("orderer".equals(type)) {
+                clientCert = Paths.get(getTestChannelPath(), "crypto-config/ordererOrganizations/example.com/users/Admin@example.com/tls/client.crt").toFile();
+
+                clientKey = Paths.get(getTestChannelPath(), "crypto-config/ordererOrganizations/example.com/users/Admin@example.com/tls/client.key").toFile();
+            } else {
+                clientCert = Paths.get(getTestChannelPath(), "crypto-config/peerOrganizations/", domainName, "users/User1@" + domainName, "tls/client.crt").toFile();
+                clientKey = Paths.get(getTestChannelPath(), "crypto-config/peerOrganizations/", domainName, "users/User1@" + domainName, "tls/client.key").toFile();
+            }
+
+            if (!clientCert.exists()) {
+                throw new RuntimeException(String.format("Missing  client cert file for: %s. Could not find at location: %s", name,
+                        clientCert.getAbsolutePath()));
+            }
+
+            if (!clientKey.exists()) {
+                throw new RuntimeException(String.format("Missing  client key file for: %s. Could not find at location: %s", name,
+                        clientKey.getAbsolutePath()));
+            }
+            ret.setProperty("clientCertFile", clientCert.getAbsolutePath());
+            ret.setProperty("clientKeyFile", clientKey.getAbsolutePath());
+        }
+
         ret.setProperty("pemFile", cert.getAbsolutePath());
-        //      ret.setProperty("trustServerCertificate", "true"); //testing environment only NOT FOR PRODUCTION!
+
         ret.setProperty("hostnameOverride", name);
         ret.setProperty("sslProvider", "openSSL");
         ret.setProperty("negotiationType", "TLS");

src/test/java/org/hyperledger/fabric/sdkintegration/End2endAndBackAgainIT.java

@@ -220,8 +220,6 @@ public void setup() throws Exception {
             setupUsers(sampleStore);
             runFabricTest(sampleStore);
 
-
-
         } catch (Exception e) {
             e.printStackTrace();
             fail(e.getMessage());
@@ -617,35 +615,19 @@ private Channel reconstructChannel(String name, HFClient client, SampleOrg sampl
             out("Retrieved channel %s from sample store.", name);
 
         } else {
-            // foo channel do manual reconstruction.
-
-            Properties clientTLSProperties = new Properties();
-
-            final String clientPEMTLSCertificate = sampleStore.getClientPEMTLSCertificate(sampleOrg);
-            if (clientPEMTLSCertificate != null) {
-                clientTLSProperties.put("clientCertBytes", clientPEMTLSCertificate.getBytes(UTF_8));
-            }
-            final String clientPEMTLSKey = sampleStore.getClientPEMTLSKey(sampleOrg);
-
-            if (clientPEMTLSKey != null) {
-                clientTLSProperties.put("clientKeyBytes", clientPEMTLSKey.getBytes(UTF_8));
-            }
 
             newChannel = client.newChannel(name);
 
             for (String ordererName : sampleOrg.getOrdererNames()) {
-                Properties ordererProperties = (Properties) clientTLSProperties.clone();
-                ordererProperties.putAll(testConfig.getOrdererProperties(ordererName));
                 newChannel.addOrderer(client.newOrderer(ordererName, sampleOrg.getOrdererLocation(ordererName),
-                        ordererProperties));
+                        testConfig.getOrdererProperties(ordererName)));
             }
 
             boolean everyOther = false;
 
             for (String peerName : sampleOrg.getPeerNames()) {
                 String peerLocation = sampleOrg.getPeerLocation(peerName);
                 Properties peerProperties = testConfig.getPeerProperties(peerName);
-                peerProperties.putAll(clientTLSProperties);
                 Peer peer = client.newPeer(peerName, peerLocation, peerProperties);
                 final PeerOptions peerEventingOptions = // we have two peers on one use block on other use filtered
                         everyOther ?
@@ -665,10 +647,8 @@ private Channel reconstructChannel(String name, HFClient client, SampleOrg sampl
                 //Should have two peers with all roles but event source.
                 assertEquals(2, newChannel.getPeers(PeerRole.NO_EVENT_SOURCE).size());
                 for (String eventHubName : sampleOrg.getEventHubNames()) {
-                    Properties eventhubProperties = (Properties) clientTLSProperties.clone();
-                    eventhubProperties.putAll(testConfig.getEventHubProperties(eventHubName));
                     EventHub eventHub = client.newEventHub(eventHubName, sampleOrg.getEventHubLocation(eventHubName),
-                            eventhubProperties);
+                            testConfig.getEventHubProperties(eventHubName));
                     newChannel.addEventHub(eventHub);
                 }
             } else {

src/test/java/org/hyperledger/fabric/sdkintegration/End2endIT.java

@@ -807,9 +807,6 @@ Channel constructChannel(String name, HFClient client, SampleOrg sampleOrg) thro
             ordererProperties.put("grpc.NettyChannelBuilderOption.keepAliveTimeout", new Object[] {8L, TimeUnit.SECONDS});
             ordererProperties.put("grpc.NettyChannelBuilderOption.keepAliveWithoutCalls", new Object[] {true});
 
-            if (!clientTLSProperties.isEmpty()) {
-                ordererProperties.putAll(clientTLSProperties.get(sampleOrg.getName()));
-            }
 
             orderers.add(client.newOrderer(orderName, sampleOrg.getOrdererLocation(orderName),
                     ordererProperties));
@@ -836,9 +833,6 @@ Channel constructChannel(String name, HFClient client, SampleOrg sampleOrg) thro
                 peerProperties = new Properties();
             }
 
-            if (!clientTLSProperties.isEmpty()) {
-                peerProperties.putAll(clientTLSProperties.get(sampleOrg.getName()));
-            }
 
             //Example of setting specific options on grpc's NettyChannelBuilder
             peerProperties.put("grpc.NettyChannelBuilderOption.maxInboundMessageSize", 9000000);
@@ -871,9 +865,6 @@ Channel constructChannel(String name, HFClient client, SampleOrg sampleOrg) thro
             eventHubProperties.put("grpc.NettyChannelBuilderOption.keepAliveTime", new Object[] {5L, TimeUnit.MINUTES});
             eventHubProperties.put("grpc.NettyChannelBuilderOption.keepAliveTimeout", new Object[] {8L, TimeUnit.SECONDS});
 
-            if (!clientTLSProperties.isEmpty()) {
-                eventHubProperties.putAll(clientTLSProperties.get(sampleOrg.getName()));
-            }
 
             EventHub eventHub = client.newEventHub(eventHubName, sampleOrg.getEventHubLocation(eventHubName),
                     eventHubProperties);

src/test/java/org/hyperledger/fabric/sdkintegration/NetworkConfigIT.java

@@ -20,6 +20,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -103,8 +104,6 @@ public class NetworkConfigIT {
 
     private static Map<String, User> orgRegisteredUsers = new HashMap<>();
 
-
-
     @BeforeClass
     public static void doMainSetup() throws Exception {
         out("\n\n\nRUNNING: NetworkConfigIT.\n");
@@ -115,6 +114,45 @@ public static void doMainSetup() throws Exception {
         // Use the appropriate TLS/non-TLS network config file
         networkConfig = NetworkConfig.fromYamlFile(testConfig.getTestNetworkConfigFileYAML());
 
+        networkConfig.getOrdererNames().forEach(ordererName -> {
+            try {
+                Properties ordererProperties = networkConfig.getOrdererProperties(ordererName);
+                Properties testProp = testConfig.getEndPointProperties("orderer", ordererName);
+                ordererProperties.setProperty("clientCertFile", testProp.getProperty("clientCertFile"));
+                ordererProperties.setProperty("clientKeyFile", testProp.getProperty("clientKeyFile"));
+                networkConfig.setOrdererProperties(ordererName, ordererProperties);
+
+            } catch (InvalidArgumentException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        networkConfig.getPeerNames().forEach(peerName -> {
+            try {
+                Properties peerProperties = networkConfig.getPeerProperties(peerName);
+                Properties testProp = testConfig.getEndPointProperties("peer", peerName);
+                peerProperties.setProperty("clientCertFile", testProp.getProperty("clientCertFile"));
+                peerProperties.setProperty("clientKeyFile", testProp.getProperty("clientKeyFile"));
+                networkConfig.setPeerProperties(peerName, peerProperties);
+
+            } catch (InvalidArgumentException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        networkConfig.getEventHubNames().forEach(eventhubName -> {
+            try {
+                Properties eventHubsProperties = networkConfig.getEventHubsProperties(eventhubName);
+                Properties testProp = testConfig.getEndPointProperties("peer", eventhubName);
+                eventHubsProperties.setProperty("clientCertFile", testProp.getProperty("clientCertFile"));
+                eventHubsProperties.setProperty("clientKeyFile", testProp.getProperty("clientKeyFile"));
+                networkConfig.setEventHubProperties(eventhubName, eventHubsProperties);
+
+            } catch (InvalidArgumentException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
         //Check if we get access to defined CAs!
         NetworkConfig.OrgInfo org = networkConfig.getOrganizationInfo("Org1");
         CAInfo caInfo = org.getCertificateAuthorities().get(0);