From 8fb837e12e2b11417a76691d53a26e9c69685581 Mon Sep 17 00:00:00 2001 From: David Enyeart Date: Sun, 19 Jan 2025 23:22:51 -0500 Subject: [PATCH 1/2] Add gossip integration test for renewing expired cert -Test renew cert before expiration (passes) -Test renew cert after expiration (previously failed) Signed-off-by: David Enyeart --- integration/gossip/gossip_test.go | 126 ++++++++++++++++++++++++++++++ integration/nwo/network.go | 1 + 2 files changed, 127 insertions(+) diff --git a/integration/gossip/gossip_test.go b/integration/gossip/gossip_test.go index 32d156f5192..5d4c3ce0993 100644 --- a/integration/gossip/gossip_test.go +++ b/integration/gossip/gossip_test.go @@ -7,6 +7,10 @@ package gossip import ( + "crypto/ecdsa" + "crypto/rand" + "crypto/x509" + "encoding/pem" "fmt" "os" "path/filepath" @@ -285,8 +289,130 @@ var _ = Describe("Gossip State Transfer and Membership", func() { assertPeerMembershipUpdate(network, peer1Org1, []*nwo.Peer{peer0Org2, peer1Org2}, nwprocs, expectedMsgFromExpirationCallback) }) }) + + It("updates membership for a peer with a renewed certificate", func() { + network.Bootstrap() + orderer := network.Orderer("orderer") + nwprocs.ordererRunner = network.OrdererRunner(orderer) + nwprocs.ordererProcess = ifrit.Invoke(nwprocs.ordererRunner) + Eventually(nwprocs.ordererProcess.Ready(), network.EventuallyTimeout).Should(BeClosed()) + + peer0Org1 := network.Peer("Org1", "peer0") + peer0Org2 := network.Peer("Org2", "peer0") + + By("bringing up a peer in each organization") + startPeers(nwprocs, false, peer0Org1, peer0Org2) + + channelparticipation.JoinOrdererAppChannel(network, "testchannel", orderer, nwprocs.ordererRunner) + + By("joining peers to channel") + network.JoinChannel(channelName, orderer, peer0Org1, peer0Org2) + + By("verifying membership of both peers") + bothPeers := []*nwo.Peer{peer0Org1, peer0Org2} + network.VerifyMembership(bothPeers, channelName) + + fmt.Println("===MEMBERSHIP VERIFIED===") + time.Sleep(5 * time.Second) + + By("stopping, renewing peer0Org2 certificate before expiration, and restarting") + stopPeers(nwprocs, peer0Org2) + renewPeerCertificate(network, peer0Org2, time.Now().Add(time.Minute)) + + fmt.Println("===STOPPED AND RENEWED peer0Org2, RESTARTING, CHECK FOR PKI-ID REPLACEMENT===") + time.Sleep(5 * time.Second) + + startPeers(nwprocs, false, peer0Org2) + + By("ensuring that peer0Org1 replaces peer0Org2 PKI-ID") + peer0Org1Runner := nwprocs.peerRunners[peer0Org1.ID()] + Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from")) + + fmt.Println("===PKI-ID REPLACED, WAIT FOR MEMBERSHIP===") + time.Sleep(5 * time.Second) + + By("verifying membership after cert renewed") + network.VerifyMembership(bothPeers, channelName) + + fmt.Println("===MEMBERSHIP VERIFIED WITH RENEWED CERT, WAIT FOR CERT TO EXPIRE AND THEN RENEW AGAIN ===") + time.Sleep(5 * time.Second) + + By("waiting for cert to expire within a minute") + Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("gossipping peer identity expired")) + + By("stopping, renewing peer0Org2 certificate again after its expiration, restarting") + stopPeers(nwprocs, peer0Org2) + renewPeerCertificate(network, peer0Org2, time.Now().Add(time.Hour)) + + fmt.Println("===STOPPED AND RENEWED peer0Org2 AGAIN AFTER EXPIRATION, RESTARTING, CHECK FOR PKI-ID REPLACEMENT AGAIN===") + time.Sleep(5 * time.Second) + + startPeers(nwprocs, false, peer0Org2) + + By("ensuring that peer0Org1 replaces peer0Org2 PKI-ID after it expired") + Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from")) + }) }) +// renewPeerCertificate renews the certificate with a given expirationTime and re-writes it to the peer's signcert directory +func renewPeerCertificate(network *nwo.Network, peer *nwo.Peer, expirationTime time.Time) { + peerDomain := network.Organization(peer.Organization).Domain + + peerCAKeyPath := filepath.Join(network.RootDir, "crypto", "peerOrganizations", peerDomain, "ca", "priv_sk") + peerCAKey, err := os.ReadFile(peerCAKeyPath) + Expect(err).NotTo(HaveOccurred()) + + peerCACertPath := filepath.Join(network.RootDir, "crypto", "peerOrganizations", peerDomain, "ca", fmt.Sprintf("ca.%s-cert.pem", peerDomain)) + peerCACert, err := os.ReadFile(peerCACertPath) + Expect(err).NotTo(HaveOccurred()) + + peerCertPath := filepath.Join(network.PeerLocalMSPDir(peer), "signcerts", fmt.Sprintf("peer0.%s-cert.pem", peerDomain)) + peerCert, err := os.ReadFile(peerCertPath) + Expect(err).NotTo(HaveOccurred()) + + renewedCert, _ := expireCertificate(peerCert, peerCACert, peerCAKey, expirationTime) + err = os.WriteFile(peerCertPath, renewedCert, 0o600) + fmt.Println(peerCertPath) + Expect(err).NotTo(HaveOccurred()) +} + +// expireCertificate re-creates and re-signs a certificate with a new expirationTime +func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time.Time) (expiredcertPEM []byte, earlyMadeCACertPEM []byte) { + keyAsDER, _ := pem.Decode(caKeyPEM) + caKeyWithoutType, err := x509.ParsePKCS8PrivateKey(keyAsDER.Bytes) + Expect(err).NotTo(HaveOccurred()) + caKey := caKeyWithoutType.(*ecdsa.PrivateKey) + + caCertAsDER, _ := pem.Decode(caCertPEM) + caCert, err := x509.ParseCertificate(caCertAsDER.Bytes) + Expect(err).NotTo(HaveOccurred()) + + certAsDER, _ := pem.Decode(certPEM) + cert, err := x509.ParseCertificate(certAsDER.Bytes) + Expect(err).NotTo(HaveOccurred()) + + cert.Raw = nil + caCert.Raw = nil + // The certificate was made 1 minute ago (1 hour doesn't work since cert will be before original CA cert NotBefore time) + cert.NotBefore = time.Now().Add((-1) * time.Minute) + // As well as the CA certificate + caCert.NotBefore = time.Now().Add((-1) * time.Minute) + // The certificate expires now + cert.NotAfter = expirationTime + + // The CA signs the certificate + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + // The CA signs its own certificate + caCertBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caCert.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + expiredcertPEM = pem.EncodeToMemory(&pem.Block{Bytes: certBytes, Type: "CERTIFICATE"}) + earlyMadeCACertPEM = pem.EncodeToMemory(&pem.Block{Bytes: caCertBytes, Type: "CERTIFICATE"}) + return +} + func runTransactions(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, chaincodeName string, channelID string) { for i := 0; i < 5; i++ { sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{ diff --git a/integration/nwo/network.go b/integration/nwo/network.go index d81652490d9..87c5d2806c1 100644 --- a/integration/nwo/network.go +++ b/integration/nwo/network.go @@ -1211,6 +1211,7 @@ func (n *Network) ConfigTxGen(command Command) (*gexec.Session, error) { func (n *Network) Discover(command Command) (*gexec.Session, error) { cmd := NewCommand(n.Components.Discover(), command) cmd.Args = append(cmd.Args, "--peerTLSCA", n.CACertsBundlePath()) + cmd.Env = []string{"FABRIC_LOGGING_SPEC=info:grpc=warn"} // suppress chatty grpc info messages return n.StartSession(cmd, command.SessionName()) } From 6142f168c1d2ae9d4c24d7fa83ef83f15062822f Mon Sep 17 00:00:00 2001 From: David Enyeart Date: Thu, 23 Jan 2025 01:10:26 -0500 Subject: [PATCH 2/2] Fix gossip membership after cert expires If a peer's cert expires while it is still in gossip memory for another peer, membership cannot be re-established after the cert is renewed. The fix is to acknowledge that the expired cert's peer is no longer in gossip identity store and accept new connection with the new PKI-ID and identity. Signed-off-by: David Enyeart --- gossip/comm/comm_impl.go | 8 ++++ integration/gossip/gossip_test.go | 76 ++++++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/gossip/comm/comm_impl.go b/gossip/comm/comm_impl.go index 6a443ae0cbe..8796eb46244 100644 --- a/gossip/comm/comm_impl.go +++ b/gossip/comm/comm_impl.go @@ -162,6 +162,14 @@ func (c *commImpl) createConnection(endpoint string, expectedPKIID common.PKIidT return nil, errors.WithStack(err) } + if len(expectedPKIID) > 0 { + identity, _ := c.idMapper.Get(expectedPKIID) + if len(identity) == 0 { + c.logger.Warningf("Identity of %x is no longer found in the identity store, aborting connection", expectedPKIID) + return nil, errors.New("identity no longer recognized") + } + } + ctx, cancel = context.WithCancel(context.Background()) if stream, err = cl.GossipStream(ctx); err == nil { connInfo, err = c.authenticateRemotePeer(stream, true, false) diff --git a/integration/gossip/gossip_test.go b/integration/gossip/gossip_test.go index 5d4c3ce0993..be4c0d442e3 100644 --- a/integration/gossip/gossip_test.go +++ b/integration/gossip/gossip_test.go @@ -9,9 +9,11 @@ package gossip import ( "crypto/ecdsa" "crypto/rand" + "crypto/sha256" "crypto/x509" "encoding/pem" "fmt" + "math/big" "os" "path/filepath" "syscall" @@ -309,8 +311,11 @@ var _ = Describe("Gossip State Transfer and Membership", func() { network.JoinChannel(channelName, orderer, peer0Org1, peer0Org2) By("verifying membership of both peers") - bothPeers := []*nwo.Peer{peer0Org1, peer0Org2} - network.VerifyMembership(bothPeers, channelName) + // bothPeers := []*nwo.Peer{peer0Org1, peer0Org2} + // network.VerifyMembership(bothPeers, channelName) + // expectedPeers := make([]nwo.DiscoveredPeer, 1) + // expectedPeers[0] = network.DiscoveredPeer(peer0Org2, "_lifecycle") + Eventually(nwo.DiscoverPeers(network, peer0Org1, "User1", "testchannel"), 50*time.Second, 100*time.Millisecond).Should(ContainElements(network.DiscoveredPeer(peer0Org2, "_lifecycle"))) fmt.Println("===MEMBERSHIP VERIFIED===") time.Sleep(5 * time.Second) @@ -328,29 +333,39 @@ var _ = Describe("Gossip State Transfer and Membership", func() { peer0Org1Runner := nwprocs.peerRunners[peer0Org1.ID()] Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from")) - fmt.Println("===PKI-ID REPLACED, WAIT FOR MEMBERSHIP===") + fmt.Println("===PKI-ID REPLACED===") time.Sleep(5 * time.Second) By("verifying membership after cert renewed") - network.VerifyMembership(bothPeers, channelName) + Eventually( + nwo.DiscoverPeers(network, peer0Org1, "User1", "testchannel"), + 60*time.Second, + 100*time.Millisecond). + Should(ContainElements(network.DiscoveredPeer(network.Peer("Org2", "peer0"), "_lifecycle"))) - fmt.Println("===MEMBERSHIP VERIFIED WITH RENEWED CERT, WAIT FOR CERT TO EXPIRE AND THEN RENEW AGAIN ===") + fmt.Println("===MEMBERSHIP VERIFIED WITH RENEWED CERT, NOW WAIT FOR CERT TO EXPIRE===") time.Sleep(5 * time.Second) By("waiting for cert to expire within a minute") Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("gossipping peer identity expired")) - By("stopping, renewing peer0Org2 certificate again after its expiration, restarting") + By("stopping, renewing peer0Org2 certificate again after its expiration") stopPeers(nwprocs, peer0Org2) renewPeerCertificate(network, peer0Org2, time.Now().Add(time.Hour)) - fmt.Println("===STOPPED AND RENEWED peer0Org2 AGAIN AFTER EXPIRATION, RESTARTING, CHECK FOR PKI-ID REPLACEMENT AGAIN===") + fmt.Println("===STOPPED AND RENEWED peer0Org2 AGAIN AFTER EXPIRATION, RESTARTING, CHECK FOR MEMBERSHIP AGAIN===") time.Sleep(5 * time.Second) + By("ensuring that peer0Org1 establishes membership with peer0Org2 after final restart post-expiration") startPeers(nwprocs, false, peer0Org2) - By("ensuring that peer0Org1 replaces peer0Org2 PKI-ID after it expired") - Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from")) + Eventually( + nwo.DiscoverPeers(network, peer0Org1, "User1", "testchannel"), + 60*time.Second, + 100*time.Millisecond). + Should(ContainElements(network.DiscoveredPeer(network.Peer("Org2", "peer0"), "_lifecycle"))) + + time.Sleep(300 * time.Minute) }) }) @@ -400,8 +415,31 @@ func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time. // The certificate expires now cert.NotAfter = expirationTime - // The CA signs the certificate - certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey) + // The CA creates and signs a temporary certificate + tempCertBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + // Force the certificate to use Low-S signature to be compatible with the identities that Fabric uses + + // Parse the certificate to extract the TBS (to-be-signed) data + tempParsedCert, err := x509.ParseCertificate(tempCertBytes) + Expect(err).NotTo(HaveOccurred()) + + // Hash the TBS data + hash := sha256.Sum256(tempParsedCert.RawTBSCertificate) + + // Sign the hash using forceLowS + r, s, err := forceLowS(caKey, hash[:]) + Expect(err).NotTo(HaveOccurred()) + + // Encode the signature (DER format) + signature := append(r.Bytes(), s.Bytes()...) + + // Replace the signature in the certificate with the low-s signature + tempParsedCert.Signature = signature + + // Re-encode the certificate with the low-s signature + certBytes, err := x509.CreateCertificate(rand.Reader, tempParsedCert, caCert, cert.PublicKey, caKey) Expect(err).NotTo(HaveOccurred()) // The CA signs its own certificate @@ -413,6 +451,22 @@ func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time. return } +// forceLowS ensures the ECDSA signature's S value is low +func forceLowS(priv *ecdsa.PrivateKey, hash []byte) (r, s *big.Int, err error) { + r, s, err = ecdsa.Sign(rand.Reader, priv, hash) + Expect(err).NotTo(HaveOccurred()) + + curveOrder := priv.Curve.Params().N + halfOrder := new(big.Int).Rsh(curveOrder, 1) // curveOrder / 2 + + // If s is greater than half the order, replace it with curveOrder - s + if s.Cmp(halfOrder) > 0 { + s.Sub(curveOrder, s) + } + + return r, s, nil +} + func runTransactions(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, chaincodeName string, channelID string) { for i := 0; i < 5; i++ { sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{