From 9788c9b3f924543b2a3b72b8480c2f70bfc3a70d Mon Sep 17 00:00:00 2001 From: Julian Castrence Date: Tue, 10 Aug 2021 15:58:02 -0400 Subject: [PATCH] Early Differences Flag Signed-off-by: Julian Castrence --- cmd/ledgerutil/main.go | 23 ++-- internal/ledgerutil/compare.go | 160 +++++++++++++++++++------ internal/ledgerutil/compare_test.go | 177 ++++++++++++++++++++++++---- 3 files changed, 284 insertions(+), 76 deletions(-) diff --git a/cmd/ledgerutil/main.go b/cmd/ledgerutil/main.go index 27211b4406a..92f01bd4bd9 100644 --- a/cmd/ledgerutil/main.go +++ b/cmd/ledgerutil/main.go @@ -9,7 +9,6 @@ package main import ( "fmt" "os" - "path/filepath" "github.com/hyperledger/fabric/internal/ledgerutil" "gopkg.in/alecthomas/kingpin.v2" @@ -17,6 +16,11 @@ import ( const ( compareErrorMessage = "Ledger Compare Error: " + outputDirDesc = "Snapshot comparison json results output directory. Default is the current directory." + firstDiffsDesc = "Number of differences to record in " + ledgerutil.FirstDiffsByHeight + + ". Generating a file of more than the first 10 differences will result in a large amount " + + "of memory usage and is not recommended. Defaults to 10. If set to 0, will not produce " + + ledgerutil.FirstDiffsByHeight + "." ) var ( @@ -25,7 +29,8 @@ var ( compare = app.Command("compare", "Compare two ledgers via their snapshots.") snapshotPath1 = compare.Arg("snapshotPath1", "First ledger snapshot directory.").Required().String() snapshotPath2 = compare.Arg("snapshotPath2", "Second ledger snapshot directory.").Required().String() - outputDir = compare.Flag("outputDir", "Snapshot comparison json results output directory. Default is the current directory.").Short('o').String() + outputDir = compare.Flag("outputDir", outputDirDesc).Short('o').String() + firstDiffs = compare.Flag("firstDiffs", firstDiffsDesc).Short('f').Default("10").Int() troubleshoot = app.Command("troubleshoot", "Identify potentially divergent transactions.") @@ -46,8 +51,6 @@ func main() { case compare.FullCommand(): - cleanpath := filepath.Clean(*outputDir) - // Determine result json file location if *outputDir == "" { *outputDir, err = os.Getwd() @@ -55,17 +58,9 @@ func main() { fmt.Printf("%s%s\n", compareErrorMessage, err) return } - cleanpath = "the current directory" - } - // Clean output - switch cleanpath { - case ".": - cleanpath = "the current directory" - case "..": - cleanpath = "the parent directory" } - count, err := ledgerutil.Compare(*snapshotPath1, *snapshotPath2, *outputDir) + count, outputDirPath, err := ledgerutil.Compare(*snapshotPath1, *snapshotPath2, *outputDir, *firstDiffs) if err != nil { fmt.Printf("%s%s\n", compareErrorMessage, err) return @@ -75,7 +70,7 @@ func main() { if count == -1 { fmt.Println("Both snapshot public state hashes were the same. No results were generated.") } else { - fmt.Printf("Results saved to %s. Total differences found: %d\n", cleanpath, count) + fmt.Printf("Results saved to %s. Total differences found: %d\n", outputDirPath, count) } case troubleshoot.FullCommand(): diff --git a/internal/ledgerutil/compare.go b/internal/ledgerutil/compare.go index 2b27cfb3b7e..a6e9804fd58 100644 --- a/internal/ledgerutil/compare.go +++ b/internal/ledgerutil/compare.go @@ -14,6 +14,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strings" "github.com/golang/protobuf/proto" @@ -27,32 +28,36 @@ import ( const ( // AllDiffsByKey - Filename for the json output that contains all differences ordered by key AllDiffsByKey = "all_diffs_by_key.json" + // FirstDiffsByHeight - Filename for the json output that contains the first n differences ordered by height + FirstDiffsByHeight = "first_diffs_by_height.json" ) // Compare - Compares two ledger snapshots and outputs the differences in snapshot records // This function will throw an error if the output directory already exist in the outputDirLoc // Function will return count of -1 if the public state hashes are the same -func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string) (count int, err error) { +func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firstDiffs int) (count int, outputDirPath string, err error) { + var records diffRecordSlice + // Check the hashes between two files hashPath1 := filepath.Join(snapshotDir1, kvledger.SnapshotSignableMetadataFileName) hashPath2 := filepath.Join(snapshotDir2, kvledger.SnapshotSignableMetadataFileName) equal, channelName, blockHeight, err := snapshotsComparable(hashPath1, hashPath2) if err != nil { - return 0, err + return 0, "", err } - + // Snapshot hashes are the same if equal { - return -1, nil + return -1, "", nil } // Output directory creation outputDirName := fmt.Sprintf("%s_%d_comparison", channelName, blockHeight) - outputDirPath := filepath.Join(outputDirLoc, outputDirName) + outputDirPath = filepath.Join(outputDirLoc, outputDirName) empty, err := fileutil.CreateDirIfMissing(outputDirPath) if err != nil { - return 0, err + return 0, "", err } if !empty { switch outputDirLoc { @@ -61,35 +66,35 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string) (cou case "..": outputDirLoc = "the parent directory" } - return 0, errors.Errorf("%s already exists in %s. Choose a different location or remove the existing results. Aborting compare", outputDirName, outputDirLoc) + return 0, "", errors.Errorf("%s already exists in %s. Choose a different location or remove the existing results. Aborting compare", outputDirName, outputDirLoc) } - // Create the output file - jsonOutputFile, err := newJSONFileWriter(filepath.Join(outputDirPath, AllDiffsByKey)) + // Create the output files + allOutputFile, err := newJSONFileWriter(filepath.Join(outputDirPath, AllDiffsByKey)) if err != nil { - return 0, err + return 0, "", err } // Create snapshot readers to read both snapshots snapshotReader1, err := privacyenabledstate.NewSnapshotReader(snapshotDir1, privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) if err != nil { - return 0, err + return 0, "", err } snapshotReader2, err := privacyenabledstate.NewSnapshotReader(snapshotDir2, privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) if err != nil { - return 0, err + return 0, "", err } // Read each snapshot record to begin looking for differences namespace1, snapshotRecord1, err := snapshotReader1.Next() if err != nil { - return 0, err + return 0, "", err } namespace2, snapshotRecord2, err := snapshotReader2.Next() if err != nil { - return 0, err + return 0, "", err } // Main snapshot record comparison loop @@ -107,56 +112,65 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string) (cou // Keys are the same but records are different diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, snapshotRecord2) if err != nil { - return 0, err + return 0, "", err } // Add difference to output JSON file - err = jsonOutputFile.addRecord(*diffRecord) + err = allOutputFile.addRecord(*diffRecord) if err != nil { - return 0, err + return 0, "", err + } + if firstDiffs != 0 { + records = append(records, diffRecord) } } // Advance both snapshot readers namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, err + return 0, "", err } namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, err + return 0, "", err } case 1: // Key 1 is bigger, snapshot 1 is missing a record // Snapshot 2 has the missing record, add missing to output JSON file diffRecord, err := newDiffRecord(namespace2, nil, snapshotRecord2) if err != nil { - return 0, err + return 0, "", err } // Add missing record to output JSON file - err = jsonOutputFile.addRecord(*diffRecord) + err = allOutputFile.addRecord(*diffRecord) if err != nil { - return 0, err + return 0, "", err + } + if firstDiffs != 0 { + records = append(records, diffRecord) } // Advance the second snapshot reader namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, err + return 0, "", err } case -1: // Key 2 is bigger, snapshot 2 is missing a record // Snapshot 1 has the missing record, add missing to output JSON file diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, nil) if err != nil { - return 0, err + return 0, "", err } // Add missing record to output JSON file - err = jsonOutputFile.addRecord(*diffRecord) + err = allOutputFile.addRecord(*diffRecord) if err != nil { - return 0, err + return 0, "", err + } + if firstDiffs != 0 { + records = append(records, diffRecord) } // Advance the first snapshot reader namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, err + return 0, "", err } default: @@ -172,15 +186,18 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string) (cou // Add missing to output JSON file diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, nil) if err != nil { - return 0, err + return 0, "", err } - err = jsonOutputFile.addRecord(*diffRecord) + err = allOutputFile.addRecord(*diffRecord) if err != nil { - return 0, err + return 0, "", err + } + if firstDiffs != 0 { + records = append(records, diffRecord) } namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, err + return 0, "", err } } @@ -189,24 +206,64 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string) (cou // Add missing to output JSON file diffRecord, err := newDiffRecord(namespace2, nil, snapshotRecord2) if err != nil { - return 0, err + return 0, "", err } - err = jsonOutputFile.addRecord(*diffRecord) + err = allOutputFile.addRecord(*diffRecord) if err != nil { - return 0, err + return 0, "", err + } + if firstDiffs != 0 { + records = append(records, diffRecord) } namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, err + return 0, "", err } } } - err = jsonOutputFile.close() + err = allOutputFile.close() if err != nil { - return 0, err + return 0, "", err } - return jsonOutputFile.count, nil + + // Create early differences output file + if firstDiffs != 0 { + firstDiffsOutputFile, err := newJSONFileWriter(filepath.Join(outputDirPath, FirstDiffsByHeight)) + if err != nil { + return 0, "", err + } + sort.Sort(records) + for i := 0; i < firstDiffs && i < len(records); i++ { + firstDiffsOutputFile.addRecord(records[i]) + } + err = firstDiffsOutputFile.close() + if err != nil { + return 0, "", err + } + } + + return allOutputFile.count, outputDirPath, nil +} + +type diffRecordSlice []*diffRecord + +func (s diffRecordSlice) Len() int { + return len(s) +} + +func (s diffRecordSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s diffRecordSlice) Less(i, j int) bool { + iBlockNum, iTxNum := s[i].getHeight() + jBlockNum, jTxNum := s[j].getHeight() + + if iBlockNum == jBlockNum { + return iTxNum <= jTxNum + } + return iBlockNum < jBlockNum } // diffRecord represents a diverging record in json @@ -249,6 +306,12 @@ func newDiffRecord(namespace string, record1 *privacyenabledstate.SnapshotRecord }, nil } +// Get height from a diffRecord +func (d diffRecord) getHeight() (blockNum uint64, txNum uint64) { + r := earlierRecord(d.Record1, d.Record2) + return r.BlockNum, r.TxNum +} + // snapshotRecord represents the data of a snapshot record in json type snapshotRecord struct { Value string `json:"value"` @@ -256,6 +319,27 @@ type snapshotRecord struct { TxNum uint64 `json:"txNum"` } +func earlierRecord(r1 *snapshotRecord, r2 *snapshotRecord) *snapshotRecord { + if r1 == nil { + return r2 + } + if r2 == nil { + return r1 + } + // Determine earlier record by block height + if r1.BlockNum < r2.BlockNum { + return r1 + } + if r2.BlockNum < r1.BlockNum { + return r2 + } + // Record block heights are the same, determine earlier transaction + if r1.TxNum < r2.TxNum { + return r1 + } + return r2 +} + // Creates a new SnapshotRecord func newSnapshotRecord(record *privacyenabledstate.SnapshotRecord) (*snapshotRecord, error) { blockNum, txNum, err := heightFromBytes(record.Version) diff --git a/internal/ledgerutil/compare_test.go b/internal/ledgerutil/compare_test.go index 615bafe1bf4..b88d575ae0a 100644 --- a/internal/ledgerutil/compare_test.go +++ b/internal/ledgerutil/compare_test.go @@ -102,6 +102,25 @@ func TestCompare(t *testing.T) { }, } + sampleRecords5 := []*testRecord{ + { + namespace: "ns1", key: "k1", value: "v1", + blockNum: 1, txNum: 1, metadata: "md1", + }, + { + namespace: "ns1", key: "k2", value: "v3", + blockNum: 1, txNum: 1, metadata: "md2", + }, + { + namespace: "ns3", key: "k1", value: "v4", + blockNum: 2, txNum: 1, metadata: "md4", + }, + { + namespace: "ns3", key: "k2", value: "v5", + blockNum: 1, txNum: 3, metadata: "md5", + }, + } + // Signable metadata samples for snapshots sampleSignableMetadata1 := &kvledger.SnapshotSignableMetadata{ ChannelName: "testchannel", @@ -238,13 +257,101 @@ func TestCompare(t *testing.T) { ]` expectedDiffDatabaseError := "the supplied snapshots appear to be non-comparable. State db types do not match." + "\nSnapshot1 state db type: testdatabase\nSnapshot2 state db type: testdatabase2" + expectedFirstDiffs := `[ + { + "namespace" : "ns1", + "key" : "k2", + "snapshotrecord1" : { + "value" : "v2", + "blockNum" : 1, + "txNum" : 1 + }, + "snapshotrecord2" : { + "value" : "v3", + "blockNum" : 1, + "txNum" : 1 + } + }, + { + "namespace" : "ns2", + "key" : "k1", + "snapshotrecord1" : { + "value" : "v3", + "blockNum" : 1, + "txNum" : 2 + }, + "snapshotrecord2" : null + }, + { + "namespace" : "ns3", + "key" : "k2", + "snapshotrecord1" : null, + "snapshotrecord2" : { + "value" : "v5", + "blockNum" : 1, + "txNum" : 3 + } + } + ]` + expectedAllDiffs := `[ + { + "namespace" : "ns1", + "key" : "k2", + "snapshotrecord1" : { + "value" : "v2", + "blockNum" : 1, + "txNum" : 1 + }, + "snapshotrecord2" : { + "value" : "v3", + "blockNum" : 1, + "txNum" : 1 + } + }, + { + "namespace" : "ns2", + "key" : "k1", + "snapshotrecord1" : { + "value" : "v3", + "blockNum" : 1, + "txNum" : 2 + }, + "snapshotrecord2" : null + }, + { + "namespace" : "ns3", + "key" : "k1", + "snapshotrecord1" : { + "value" : "v4", + "blockNum" : 2, + "txNum" : 0 + }, + "snapshotrecord2" : { + "value" : "v4", + "blockNum" : 2, + "txNum" : 1 + } + }, + { + "namespace" : "ns3", + "key" : "k2", + "snapshotrecord1" : null, + "snapshotrecord2" : { + "value" : "v5", + "blockNum" : 1, + "txNum" : 3 + } + } + ]` testCases := map[string]struct { inputTestRecords1 []*testRecord inputSignableMetadata1 *kvledger.SnapshotSignableMetadata inputTestRecords2 []*testRecord inputSignableMetadata2 *kvledger.SnapshotSignableMetadata - expectedOutput string + expectedAllOutput string + expectedFirstOutput string + firstN int expectedOutputType string expectedDiffCount int }{ @@ -254,7 +361,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords2, inputSignableMetadata2: sampleSignableMetadata2, - expectedOutput: expectedDifferenceResult, + expectedAllOutput: expectedDifferenceResult, expectedOutputType: "json", expectedDiffCount: 1, }, @@ -264,7 +371,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords3, inputSignableMetadata2: sampleSignableMetadata2, - expectedOutput: expectedMissingResult1, + expectedAllOutput: expectedMissingResult1, expectedOutputType: "json", expectedDiffCount: 1, }, @@ -274,7 +381,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata2, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedOutput: expectedMissingResult2, + expectedAllOutput: expectedMissingResult2, expectedOutputType: "json", expectedDiffCount: 1, }, @@ -284,7 +391,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords4, inputSignableMetadata2: sampleSignableMetadata2, - expectedOutput: expectedMissingTailResult1, + expectedAllOutput: expectedMissingTailResult1, expectedOutputType: "json", expectedDiffCount: 2, }, @@ -294,7 +401,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata2, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedOutput: expectedMissingTailResult2, + expectedAllOutput: expectedMissingTailResult2, expectedOutputType: "json", expectedDiffCount: 2, }, @@ -304,7 +411,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedOutput: "", + expectedAllOutput: "", expectedOutputType: "same-hash", expectedDiffCount: -1, }, @@ -314,7 +421,7 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords2, inputSignableMetadata2: sampleSignableMetadata3, - expectedOutput: expectedDiffDatabaseError, + expectedAllOutput: expectedDiffDatabaseError, expectedOutputType: "error", expectedDiffCount: 0, }, @@ -327,6 +434,18 @@ func TestCompare(t *testing.T) { expectedOutputType: "exists-error", expectedDiffCount: 0, }, + // User requested first 3 differences + "first-3-diffs": { + inputTestRecords1: sampleRecords1, + inputSignableMetadata1: sampleSignableMetadata1, + inputTestRecords2: sampleRecords5, + inputSignableMetadata2: sampleSignableMetadata2, + expectedAllOutput: expectedAllDiffs, + expectedFirstOutput: expectedFirstDiffs, + firstN: 3, + expectedOutputType: "json", + expectedDiffCount: 4, + }, } // Run test cases individually @@ -350,20 +469,24 @@ func TestCompare(t *testing.T) { require.NoError(t, err) // Compare snapshots and check the output - count, out, err := compareSnapshots(snapshotDir1, snapshotDir2, resultsDir) + count, allOut, firstOut, err := compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) switch testCase.expectedOutputType { case "error": require.Equal(t, testCase.expectedDiffCount, count) - require.ErrorContains(t, err, testCase.expectedOutput) + require.ErrorContains(t, err, testCase.expectedAllOutput) case "exists-error": require.NoError(t, err) - count, _, err = compareSnapshots(snapshotDir1, snapshotDir2, resultsDir) + count, _, _, err = compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) require.Equal(t, testCase.expectedDiffCount, count) require.ErrorContains(t, err, "testchannel_2_comparison already exists in "+resultsDir+". Choose a different location or remove the existing results. Aborting compare") case "json": require.Equal(t, testCase.expectedDiffCount, count) require.NoError(t, err) - require.JSONEq(t, testCase.expectedOutput, out) + require.JSONEq(t, testCase.expectedAllOutput, allOut) + if firstOut != "" { + require.JSONEq(t, testCase.expectedFirstOutput, firstOut) + } + case "same-hash": require.Equal(t, testCase.expectedDiffCount, count) require.NoError(t, err) @@ -421,29 +544,35 @@ func createSnapshot(dir string, pubStateRecords []*testRecord, signableMetadata } // compareSnapshots calls the Compare tool and extracts the result json -func compareSnapshots(ss1 string, ss2 string, res string) (int, string, error) { +func compareSnapshots(ss1 string, ss2 string, res string, firstN int) (int, string, string, error) { // Run compare tool on snapshots - count, err := Compare(ss1, ss2, res) + count, opath, err := Compare(ss1, ss2, res, firstN) if err != nil || count == -1 { - return count, "", err + return count, "", "", err } // Read results of output - md, err := readMetadata(filepath.Join(ss1, kvledger.SnapshotSignableMetadataFileName)) + allBytes, err := ioutil.ReadFile(filepath.Join(opath, AllDiffsByKey)) if err != nil { - return 0, "", err + return 0, "", "", err } - odir := fmt.Sprintf("%s_%d_comparison", md.ChannelName, md.LastBlockNumber) - resBytes, err := ioutil.ReadFile(filepath.Join(res, odir, AllDiffsByKey)) + allOut, err := ioutil.ReadAll(bytes.NewReader(allBytes)) if err != nil { - return 0, "", err + return 0, "", "", err } - out, err := ioutil.ReadAll(bytes.NewReader(resBytes)) - if err != nil { - return 0, "", err + if firstN != 0 { + firstBytes, err := ioutil.ReadFile(filepath.Join(opath, FirstDiffsByHeight)) + if err != nil { + return 0, "", "", err + } + firstOut, err := ioutil.ReadAll(bytes.NewReader(firstBytes)) + if err != nil { + return 0, "", "", err + } + return count, string(allOut), string(firstOut), nil } - return count, string(out), nil + return count, string(allOut), "", nil } // toBytes serializes the Height