FAB-16544 Backport orderer TLS cert overrides

This is a partial backport of the orderer endpoint TLS CA overrides
present in v2.0.  Presently, in v1.4.x because of the code structure,
it's very difficult to associate a particular TLS CA with a connection
to a particular orderer.  So, instead v1.4.x took the approach of adding
global TLS overrides.  However, moving forward into the v2.0 branch,
things have been restructured significantly and it's now possible to
bind orderer addresses and TLS CAs.

This CR takes the approach of allowing and honoring the TLS CA
overrides, but treating them in the same pooled fashion as the previous
approach.  So, althought he config binds the TLS CAs to a particular
address (which will be honored in 2.0), they are all thrown into a
global pool for v1.4.x.

Signed-off-by: Jason Yellick <[email protected]>
Change-Id: Ide0d1e75389e1373cc1d92924f01f87684aa6990

core/comm/connection.go

@@ -41,6 +41,11 @@ type PerOrgCertificateBundle map[string]CertificateBundle
 // channel --> organization --> certificates
 type OrgRootCAs map[string]PerOrgCertificateBundle
 
+type OrdererEndpoint struct {
+	Address string
+	PEMs    []byte
+}
+
 // CertificatesByChannelAndOrg returns the certificates of the given organization in the context
 // of the given channel.
 func (orc OrgRootCAs) CertificatesByChannelAndOrg(channel string, org string) CertificateBundle {
@@ -104,6 +109,7 @@ func (cs *CredentialSupport) GetDeliverServiceCredentials(
 	channelID string,
 	appendStaticRoots bool,
 	orgs []string,
+	endpointOverrides map[string]*OrdererEndpoint,
 ) (credentials.TransportCredentials, error) {
 	cs.RLock()
 	defer cs.RUnlock()
@@ -145,6 +151,10 @@ func (cs *CredentialSupport) GetDeliverServiceCredentials(
 		}
 	}
 
+	for _, override := range endpointOverrides {
+		certPool.AppendCertsFromPEM(override.PEMs)
+	}
+
 	// Finally, create a TLS client config with the computed TLS root CAs.
 	var creds credentials.TransportCredentials
 	tlsConfig := &tls.Config{

core/comm/connection_test.go

@@ -178,20 +178,20 @@ func TestCredentialSupport(t *testing.T) {
 	cs.ServerRootCAs = [][]byte{rootCAs[5]}
 	cs.ClientRootCAs = [][]byte{rootCAs[5]}
 
-	creds, _ := cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"})
+	creds, _ := cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}, nil)
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 	creds = cs.GetPeerCredentials()
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 
-	_, err := cs.GetDeliverServiceCredentials("channel99", false, nil)
+	_, err := cs.GetDeliverServiceCredentials("channel99", false, nil, nil)
 	assert.EqualError(t, err, "didn't find any root CA certs for channel channel99")
 
 	// append some bad certs and make sure things still work
 	cs.ServerRootCAs = append(cs.ServerRootCAs, []byte("badcert"))
 	cs.ServerRootCAs = append(cs.ServerRootCAs, []byte(badPEM))
-	creds, _ = cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"})
+	creds, _ = cs.GetDeliverServiceCredentials("channel1", false, []string{"SampleOrg"}, nil)
 	assert.Equal(t, "1.2", creds.Info().SecurityVersion,
 		"Expected Security version to be 1.2")
 	creds = cs.GetPeerCredentials()
@@ -284,7 +284,7 @@ func TestImpersonation(t *testing.T) {
 		AppRootCAsByChain:           make(map[string]CertificateBundle),
 		OrdererRootCAsByChainAndOrg: make(OrgRootCAs),
 	}
-	_, err := cs.GetDeliverServiceCredentials("C", false, []string{"SampleOrg"})
+	_, err := cs.GetDeliverServiceCredentials("C", false, []string{"SampleOrg"}, nil)
 	assert.Error(t, err)
 
 	cs.OrdererRootCAsByChainAndOrg.AppendCertificates("A", "SampleOrg", [][]byte{osA.caCert})
@@ -307,7 +307,7 @@ func testInvoke(
 	staticRoots bool,
 	shouldSucceed bool) {
 
-	creds, err := cs.GetDeliverServiceCredentials(channelID, staticRoots, []string{"SampleOrg"})
+	creds, err := cs.GetDeliverServiceCredentials(channelID, staticRoots, []string{"SampleOrg"}, nil)
 	assert.NoError(t, err)
 
 	endpoint := s.address

core/deliverservice/deliveryclient.go

@@ -89,7 +89,7 @@ type deliverClient struct {
 // and how it disseminates the messages to other peers
 type Config struct {
 	// ConnFactory returns a function that creates a connection to an endpoint
-	ConnFactory func(channelID string) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error)
+	ConnFactory func(channelID string, endpointOverrides map[string]*comm.OrdererEndpoint) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error)
 	// ABCFactory creates an AtomicBroadcastClient out of a connection
 	ABCFactory func(*grpc.ClientConn) orderer.AtomicBroadcastClient
 	// CryptoSvc performs cryptographic actions like message verification and signing
@@ -111,7 +111,7 @@ type ConnectionCriteria struct {
 	// OrdererEndpointOverrides specifies a map of endpoints to override.  The map
 	// key is the configured endpoint address to match and the value is the
 	// endpoint to use instead.
-	OrdererEndpointOverrides map[string]string
+	OrdererEndpointOverrides map[string]*comm.OrdererEndpoint
 }
 
 func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria {
@@ -128,7 +128,7 @@ func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria {
 		for _, endpoint := range endpoints {
 			// check if we need to override the endpoint
 			if override, ok := cc.OrdererEndpointOverrides[endpoint]; ok {
-				endpoint = override
+				endpoint = override.Address
 			}
 			res = append(res, comm.EndpointCriteria{
 				Organizations: []string{org},
@@ -145,7 +145,7 @@ func (cc ConnectionCriteria) toEndpointCriteria() []comm.EndpointCriteria {
 	for _, endpoint := range cc.OrdererEndpoints {
 		// check if we need to override the endpoint
 		if override, ok := cc.OrdererEndpointOverrides[endpoint]; ok {
-			endpoint = override
+			endpoint = override.Address
 		}
 		res = append(res, comm.EndpointCriteria{
 			Organizations: cc.Organizations,
@@ -298,13 +298,13 @@ func (d *deliverServiceImpl) newClient(chainID string, ledgerInfoProvider blocks
 		attempt := float64(attemptNum)
 		return time.Duration(math.Min(math.Pow(2, attempt)*sleepIncrement, reconnectBackoffThreshold)), true
 	}
-	connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID), d.connConfig.toEndpointCriteria())
+	connProd := comm.NewConnectionProducer(d.conf.ConnFactory(chainID, d.connConfig.OrdererEndpointOverrides), d.connConfig.toEndpointCriteria())
 	bClient := NewBroadcastClient(connProd, d.conf.ABCFactory, broadcastSetup, backoffPolicy)
 	requester.client = bClient
 	return bClient
 }
 
-func DefaultConnectionFactory(channelID string) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) {
+func DefaultConnectionFactory(channelID string, endpointOverrides map[string]*comm.OrdererEndpoint) func(endpointCriteria comm.EndpointCriteria) (*grpc.ClientConn, error) {
 	return func(criteria comm.EndpointCriteria) (*grpc.ClientConn, error) {
 		dialOpts := []grpc.DialOption{grpc.WithBlock()}
 		// set max send/recv msg sizes
@@ -323,7 +323,7 @@ func DefaultConnectionFactory(channelID string) func(endpointCriteria comm.Endpo
 		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(kaOpts)...)
 
 		if viper.GetBool("peer.tls.enabled") {
-			creds, err := comm.GetCredentialSupport().GetDeliverServiceCredentials(channelID, staticRootsEnabled(), criteria.Organizations)
+			creds, err := comm.GetCredentialSupport().GetDeliverServiceCredentials(channelID, staticRootsEnabled(), criteria.Organizations, endpointOverrides)
 			if err != nil {
 				return nil, fmt.Errorf("failed obtaining credentials for channel %s: %v", channelID, err)
 			}

core/deliverservice/deliveryclient_test.go

@@ -102,7 +102,7 @@ func TestNewDeliverService(t *testing.T) {
 		}
 	}
 
-	connFactory := func(_ string) func(comm.EndpointCriteria) (*grpc.ClientConn, error) {
+	connFactory := func(string, map[string]*comm.OrdererEndpoint) func(comm.EndpointCriteria) (*grpc.ClientConn, error) {
 		return func(endpoint comm.EndpointCriteria) (*grpc.ClientConn, error) {
 			lock.Lock()
 			defer lock.Unlock()
@@ -277,8 +277,10 @@ func TestDeliverServiceUpdateEndpoints(t *testing.T) {
 			OrdererEndpointsByOrg: map[string][]string{
 				"org1": {"localhost:5612"},
 			},
-			OrdererEndpointOverrides: map[string]string{
-				"localhost:5612": "localhost:5614",
+			OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{
+				"localhost:5612": {
+					Address: "localhost:5614",
+				},
 			},
 		},
 		deliverClients: map[string]*deliverClient{
@@ -698,7 +700,7 @@ func TestDeliverServiceBadConfig(t *testing.T) {
 }
 
 func TestRetryPolicyOverflow(t *testing.T) {
-	connFactory := func(channelID string) func(comm.EndpointCriteria) (*grpc.ClientConn, error) {
+	connFactory := func(channelID string, _ map[string]*comm.OrdererEndpoint) func(comm.EndpointCriteria) (*grpc.ClientConn, error) {
 		return func(_ comm.EndpointCriteria) (*grpc.ClientConn, error) {
 			return nil, errors.New("")
 		}
@@ -781,8 +783,10 @@ func TestToEndpointCriteria(t *testing.T) {
 			input: ConnectionCriteria{
 				Organizations:    []string{"foo", "bar"},
 				OrdererEndpoints: []string{"a", "b", "c"},
-				OrdererEndpointOverrides: map[string]string{
-					"b": "d",
+				OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{
+					"b": {
+						Address: "d",
+					},
 				},
 			},
 			expectedOut: []comm.EndpointCriteria{
@@ -818,8 +822,10 @@ func TestToEndpointCriteria(t *testing.T) {
 					"foo": {"a", "b"},
 					"bar": {"c"},
 				},
-				OrdererEndpointOverrides: map[string]string{
-					"b": "d",
+				OrdererEndpointOverrides: map[string]*comm.OrdererEndpoint{
+					"b": {
+						Address: "d",
+					},
 				},
 			},
 			expectedOut: []comm.EndpointCriteria{

core/peer/config.go

@@ -258,22 +258,36 @@ func GetClientCertificate() (tls.Certificate, error) {
 }
 
 type addressOverride struct {
-	From string `mapstructure:"from"`
-	To   string `mapstructure:"to"`
+	From        string `mapstructure:"from"`
+	To          string `mapstructure:"to"`
+	CACertsFile string `mapstructure:"caCertsFile"`
 }
 
-func GetOrdererAddressOverrides() (map[string]string, error) {
+type OrdererEndpoint struct {
+	Address string
+	PEMs    []byte
+}
+
+func GetOrdererAddressOverrides() (map[string]*comm.OrdererEndpoint, error) {
 	var overrides []addressOverride
 	err := viper.UnmarshalKey("peer.deliveryclient.addressOverrides", &overrides)
 	if err != nil {
 		return nil, err
 	}
 
-	var overrideMap map[string]string
+	var overrideMap map[string]*comm.OrdererEndpoint
 	if len(overrides) > 0 {
-		overrideMap = make(map[string]string)
+		overrideMap = make(map[string]*comm.OrdererEndpoint)
 		for _, override := range overrides {
-			overrideMap[override.From] = override.To
+			pem, err := ioutil.ReadFile(override.CACertsFile)
+			if err != nil {
+				return nil, errors.WithMessage(err, fmt.Sprintf("could not read file '%s' specified for caCertsFile of orderer endpoint override from '%s' to '%s'", override.CACertsFile, override.From, override.To))
+			}
+
+			overrideMap[override.From] = &comm.OrdererEndpoint{
+				Address: override.To,
+				PEMs:    pem,
+			}
 		}
 	}
 	return overrideMap, nil

core/peer/config_test.go

@@ -327,18 +327,52 @@ func TestGetOrdererAddressOverrides(t *testing.T) {
       addressOverrides:
         - from: myaddress
           to: youraddress
+          caCertsFile: testdata/Org1-cert.pem
         - from: myaddress2
-          to: youraddress2`
+          to: youraddress2
+          caCertsFile: testdata/Org1-cert.pem`
 
 	viper.SetConfigType("yaml")
 	err := viper.ReadConfig(bytes.NewBuffer([]byte(conf)))
 	if err != nil {
 		t.Fatalf("Failed to read test config: %s", err)
 	}
 
-	expected := map[string]string{
-		"myaddress":  "youraddress",
-		"myaddress2": "youraddress2",
+	expected := map[string]*comm.OrdererEndpoint{
+		"myaddress": {
+			Address: "youraddress",
+			PEMs: []byte(`-----BEGIN CERTIFICATE-----
+MIIB8TCCAZegAwIBAgIQU59imQ+xl+FmwuiFyUgFezAKBggqhkjOPQQDAjBYMQsw
+CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy
+YW5jaXNjbzENMAsGA1UEChMET3JnMTENMAsGA1UEAxMET3JnMTAeFw0xNzA1MDgw
+OTMwMzRaFw0yNzA1MDYwOTMwMzRaMFgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpD
+YWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRPcmcx
+MQ0wCwYDVQQDEwRPcmcxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFkpP6EqE
+87ghFi25UWLvgPatxDiYKYaVSPvpo/XDJ0+9uUmK/C2r5Bvvxx1t8eTROwN77tEK
+r+jbJIxX3ZYQMKNDMEEwDgYDVR0PAQH/BAQDAgGmMA8GA1UdJQQIMAYGBFUdJQAw
+DwYDVR0TAQH/BAUwAwEB/zANBgNVHQ4EBgQEAQIDBDAKBggqhkjOPQQDAgNIADBF
+AiEA1Xkrpq+wrmfVVuY12dJfMQlSx+v0Q3cYce9BE1i2mioCIAzqyduK/lHPI81b
+nWiU9JF9dRQ69dEV9dxd/gzamfFU
+-----END CERTIFICATE-----
+`),
+		},
+		"myaddress2": {
+			Address: "youraddress2",
+			PEMs: []byte(`-----BEGIN CERTIFICATE-----
+MIIB8TCCAZegAwIBAgIQU59imQ+xl+FmwuiFyUgFezAKBggqhkjOPQQDAjBYMQsw
+CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy
+YW5jaXNjbzENMAsGA1UEChMET3JnMTENMAsGA1UEAxMET3JnMTAeFw0xNzA1MDgw
+OTMwMzRaFw0yNzA1MDYwOTMwMzRaMFgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpD
+YWxpZm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRPcmcx
+MQ0wCwYDVQQDEwRPcmcxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFkpP6EqE
+87ghFi25UWLvgPatxDiYKYaVSPvpo/XDJ0+9uUmK/C2r5Bvvxx1t8eTROwN77tEK
+r+jbJIxX3ZYQMKNDMEEwDgYDVR0PAQH/BAQDAgGmMA8GA1UdJQQIMAYGBFUdJQAw
+DwYDVR0TAQH/BAUwAwEB/zANBgNVHQ4EBgQEAQIDBDAKBggqhkjOPQQDAgNIADBF
+AiEA1Xkrpq+wrmfVVuY12dJfMQlSx+v0Q3cYce9BE1i2mioCIAzqyduK/lHPI81b
+nWiU9JF9dRQ69dEV9dxd/gzamfFU
+-----END CERTIFICATE-----
+`),
+		},
 	}
 	overrides, err := GetOrdererAddressOverrides()
 	if err != nil {

gossip/service/gossip_service.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 
 	"github.com/hyperledger/fabric/common/metrics"
+	"github.com/hyperledger/fabric/core/comm"
 	"github.com/hyperledger/fabric/core/committer"
 	"github.com/hyperledger/fabric/core/committer/txvalidator"
 	"github.com/hyperledger/fabric/core/common/privdata"
@@ -83,7 +84,7 @@ type OrdererAddressConfig struct {
 	Addresses        []string
 	AddressesByOrg   map[string][]string
 	Organizations    []string
-	AddressOverrides map[string]string
+	AddressOverrides map[string]*comm.OrdererEndpoint
 }
 
 type privateHandler struct {