[BUG]: Topology mismatch

[Erigara]

During running iroha with periodic restarts it was noticed that iroha submits block with wrong topology.

It looks like that iroha didn't rotate full topology after the view change for whatever reason.

Issue was observed on rc19 and it's uncertain whatever it exists on latest dev branch.

Error in logs:

�[32miroha2-1  |�[0m thread 'main' panicked at 'The block should be valid in init. Blocks loaded from kura assumed to be valid: TopologyMismatch { expected: [PeerId { address: Host(SocketAddrHost { host: "iroha2-2", port: 1337 }), public_key: {digest: ed25519, payload: [67, 9D, C4, C8, 27, FB, 5B, 9A, 3A, 58, 98, E, E1, 26, 4D, 9F, 20, F, 56, 6E, 31, 2D, 1F, 71, 8A, 9C, 4B, AF, 96, C1, 2F, 32]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-1", port: 1337 }), public_key: {digest: ed25519, payload: [7C, 8E, F1, 6F, 34, 6B, 26, AA, 2C, A5, 65, 52, CD, 8B, 6D, 37, 3D, 31, CD, 42, CA, 65, 5C, 3A, 84, 3, D3, 73, F9, E0, 91, EF]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-0", port: 1337 }), public_key: {digest: ed25519, payload: [74, ED, E3, 14, EA, A8, 5, FC, F, F1, 4C, 79, 1, 46, FA, E6, E3, FC, B9, 4, 7A, 6, 91, BE, 9D, 19, 16, 9E, F3, 6C, 92, D1]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-3", port: 1337 }), public_key: {digest: ed25519, payload: [C0, BF, BA, B9, 6C, C5, AF, 53, 12, 79, B5, 0, 83, CC, 92, D8, 7, 39, 6B, B5, 35, EB, 6A, 18, B1, 5D, 35, 2C, 7C, D4, C2, 88]} }], actual: [PeerId { address: Host(SocketAddrHost { host: "iroha2-3", port: 1337 }), public_key: {digest: ed25519, payload: [C0, BF, BA, B9, 6C, C5, AF, 53, 12, 79, B5, 0, 83, CC, 92, D8, 7, 39, 6B, B5, 35, EB, 6A, 18, B1, 5D, 35, 2C, 7C, D4, C2, 88]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-2", port: 1337 }), public_key: {digest: ed25519, payload: [67, 9D, C4, C8, 27, FB, 5B, 9A, 3A, 58, 98, E, E1, 26, 4D, 9F, 20, F, 56, 6E, 31, 2D, 1F, 71, 8A, 9C, 4B, AF, 96, C1, 2F, 32]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-1", port: 1337 }), public_key: {digest: ed25519, payload: [7C, 8E, F1, 6F, 34, 6B, 26, AA, 2C, A5, 65, 52, CD, 8B, 6D, 37, 3D, 31, CD, 42, CA, 65, 5C, 3A, 84, 3, D3, 73, F9, E0, 91, EF]} }, PeerId { address: Host(SocketAddrHost { host: "iroha2-0", port: 1337 }), public_key: {digest: ed25519, payload: [74, ED, E3, 14, EA, A8, 5, FC, F, F1, 4C, 79, 1, 46, FA, E6, E3, FC, B9, 4, 7A, 6, 91, BE, 9D, 19, 16, 9E, F3, 6C, 92, D1]} }] }', core/src/sumeragi/mod.rs:246:47

Broken block and block before

Index file says there are 2712 blocks.
Printing blocks 2711-2712...
Block#2711 starts at byte offset 2676273 and is 823 bytes long.
Block#2711 :
V1(
  CommittedBlock {
      header: BlockHeader {
          timestamp: 1697153150158,
          consensus_estimation: 4000,
          height: 2711,
          view_change_index: 1,
          previous_block_hash: Some(
              { iroha_crypto::hash::HashOf<iroha_data_model::block::committed::VersionedCommittedBlock> 59a12e607032dd2d3e261159ff007efc4b95316e4fa940c40697690f1f13893d },
          ),
          transactions_hash: Some(
              { iroha_crypto::hash::HashOf<iroha_crypto::merkle::MerkleTree<iroha_data_model::transaction::VersionedSignedTransaction>> 986762a06b881dbfa5c1269dcbf9a35d39b5a1d2f45d5733479a58aff9fe8b2b },
          ),
          rejected_transactions_hash: None,
          committed_with_topology: [
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-1",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [7C, 8E, F1, 6F, 34, 6B, 26, AA, 2C, A5, 65, 52, CD, 8B, 6D, 37, 3D, 31, CD, 42, CA, 65, 5C, 3A, 84, 3, D3, 73, F9, E0, 91, EF]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-3",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [C0, BF, BA, B9, 6C, C5, AF, 53, 12, 79, B5, 0, 83, CC, 92, D8, 7, 39, 6B, B5, 35, EB, 6A, 18, B1, 5D, 35, 2C, 7C, D4, C2, 88]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-2",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [67, 9D, C4, C8, 27, FB, 5B, 9A, 3A, 58, 98, E, E1, 26, 4D, 9F, 20, F, 56, 6E, 31, 2D, 1F, 71, 8A, 9C, 4B, AF, 96, C1, 2F, 32]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-0",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [74, ED, E3, 14, EA, A8, 5, FC, F, F1, 4C, 79, 1, 46, FA, E6, E3, FC, B9, 4, 7A, 6, 91, BE, 9D, 19, 16, 9E, F3, 6C, 92, D1]},
              },
          ],
      },
      transactions: [
          TransactionValue {
              value: V1(
                  SignedTransaction {
                      signatures: iroha_crypto::signature::SignaturesOf<iroha_data_model::transaction::model::TransactionPayload> {
                          signatures: {
                              iroha_crypto::signature::SignatureOf<iroha_data_model::transaction::model::TransactionPayload>(
                                  { pub_key: ed0120DC1D41564578D64EB429C766C20D6AF75359EBC9AE831164C71060660411782C, payload: F3BA3249E7B27829C892F705221759CB2F68D75FDE4BF8A73B6731FC00550CA850740A92EB2FC66614FE05221C51FED30A52EB32A6010660D05474B324BDD909 },
                              ),
                          },
                      },
                      payload: TransactionPayload {
                          creation_time_ms: 1697153148996,
                          authority: sorabankxxx@sora,
                          instructions: [TransferBox { source_id: AssetId(sbd#cbsi#sorabankxxx@sora), object: 20.0_fx, destination_id: AccountId(opitniy_polzovatel@sora) }],
                          time_to_live_ms: Some(
                              86400000,
                          ),
                          nonce: Some(
                              1885569574,
                          ),
                          metadata: {
                              "description": "",
                          },
                      },
                  },
              ),
              error: None,
          },
      ],
      event_recommendations: [],
      signatures: iroha_crypto::signature::SignaturesOf<iroha_data_model::block::committed::model::CommittedBlock> {
          signatures: {
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed0120679DC4C827FB5B9A3A58980EE1264D9F200F566E312D1F718A9C4BAF96C12F32, payload: AE704B3AA5E9D655475D35E2E08E80BC70EF97D14148B7DB9A13D71936E18E087A9473FAE0AEB691204F4810248EFA45237E2710D58F7644E6E954F7141C5E0C },
              ),
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed01207C8EF16F346B26AA2CA56552CD8B6D373D31CD42CA655C3A8403D373F9E091EF, payload: 7D0B5964E6F2A1ED427F706329D8C2CE6AEB2E965E37D3EFC178FB5C2CC25400941DB68AE90633112D894400C04C8D94AFC0F8AAAB4D3A0128704580878FCB01 },
              ),
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed0120C0BFBAB96CC5AF531279B50083CC92D807396BB535EB6A18B15D352C7CD4C288, payload: 3F814865BFAD411C342443015813DD537C24F0935DA669ADF8083BAAE82589B70E146295515915BE90DF26F828979D4C89A6B4B5A672667467F77C5A3BC99905 },
              ),
          },
      },
  },
)
Block#2712 starts at byte offset 2677096 and is 798 bytes long.
Block#2712 :
V1(
  CommittedBlock {
      header: BlockHeader {
          timestamp: 1697153150179,
          consensus_estimation: 4000,
          height: 2712,
          view_change_index: 1,
          previous_block_hash: Some(
              { iroha_crypto::hash::HashOf<iroha_data_model::block::committed::VersionedCommittedBlock> 817a33dc80c430c6df27164ab23b23769c5ac9180376650123c88569f5558f2d },
          ),
          transactions_hash: Some(
              { iroha_crypto::hash::HashOf<iroha_crypto::merkle::MerkleTree<iroha_data_model::transaction::VersionedSignedTransaction>> ab78ee85e3e6c7c0b303e3ed3f10d92436d5defb9dd1b376a4032a28eb560661 },
          ),
          rejected_transactions_hash: None,
          committed_with_topology: [
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-3",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [C0, BF, BA, B9, 6C, C5, AF, 53, 12, 79, B5, 0, 83, CC, 92, D8, 7, 39, 6B, B5, 35, EB, 6A, 18, B1, 5D, 35, 2C, 7C, D4, C2, 88]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-2",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [67, 9D, C4, C8, 27, FB, 5B, 9A, 3A, 58, 98, E, E1, 26, 4D, 9F, 20, F, 56, 6E, 31, 2D, 1F, 71, 8A, 9C, 4B, AF, 96, C1, 2F, 32]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-1",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [7C, 8E, F1, 6F, 34, 6B, 26, AA, 2C, A5, 65, 52, CD, 8B, 6D, 37, 3D, 31, CD, 42, CA, 65, 5C, 3A, 84, 3, D3, 73, F9, E0, 91, EF]},
              },
              PeerId {
                  address: Host(
                      SocketAddrHost {
                          host: "iroha2-0",
                          port: 1337,
                      },
                  ),
                  public_key: {digest: ed25519, payload: [74, ED, E3, 14, EA, A8, 5, FC, F, F1, 4C, 79, 1, 46, FA, E6, E3, FC, B9, 4, 7A, 6, 91, BE, 9D, 19, 16, 9E, F3, 6C, 92, D1]},
              },
          ],
      },
      transactions: [
          TransactionValue {
              value: V1(
                  SignedTransaction {
                      signatures: iroha_crypto::signature::SignaturesOf<iroha_data_model::transaction::model::TransactionPayload> {
                          signatures: {
                              iroha_crypto::signature::SignatureOf<iroha_data_model::transaction::model::TransactionPayload>(
                                  { pub_key: ed01207233BFC89DCBD68C19FDE6CE6158225298EC1131B6A130D1AEB454C1AB5183C0, payload: 5B6FBB63C6FDF862634BF50EF50F8B98DBBF0A3F1D2796B224EE661DC4A02244CD3A2528E7B72103AB76F1CF13665732CADEFAB52B6999DD2231764ADA75AE0E },
                              ),
                          },
                      },
                      payload: TransactionPayload {
                          creation_time_ms: 1697153149698,
                          authority: admin@cbsi,
                          instructions: [SetKeyValueBox { object_id: AssetId(metadata##admin@cbsi), key: "block_height", value: 2710_u128 }],
                          time_to_live_ms: Some(
                              86400000,
                          ),
                          nonce: Some(
                              4183393638,
                          ),
                          metadata: {},
                      },
                  },
              ),
              error: None,
          },
      ],
      event_recommendations: [],
      signatures: iroha_crypto::signature::SignaturesOf<iroha_data_model::block::committed::model::CommittedBlock> {
          signatures: {
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed0120679DC4C827FB5B9A3A58980EE1264D9F200F566E312D1F718A9C4BAF96C12F32, payload: C73344E2DC3FF7D98D1219162FC9BA4EE6666A70A14AFCCF2C5FB706BD6A5D5530C04F45C6BBB7F698F840770860074275F7EE166C99366AEFF313DF8746C10E },
              ),
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed01207C8EF16F346B26AA2CA56552CD8B6D373D31CD42CA655C3A8403D373F9E091EF, payload: 90C6F42D9BDE07A7EF237C21753E30B2153DFB2558ABABB62D8C73BC6E5E85551722ADF99BEF92269CB28DB3642A5DCD4AFFF3513977EC3F53C756133E2FD409 },
              ),
              iroha_crypto::signature::SignatureOf<iroha_data_model::block::committed::model::CommittedBlock>(
                  { pub_key: ed0120C0BFBAB96CC5AF531279B50083CC92D807396BB535EB6A18B15D352C7CD4C288, payload: 96F0727362B3CB2AE6DCB4F1906CCD6CEA3AFDF734D9FACA940173F7E5C63A3CFB9D81BE97F092010E82E7407FAACA8C2253281172EB04A0D3E831CA4C445C00 },
              ),
          },
      },
  },
)

[AlexStroke]

https://hub.docker.com/r/erigara/iroha2/tags