From 6b92f338829e928ab24737c2c4dcdf619d8d7e6e Mon Sep 17 00:00:00 2001 From: thienlk1 Date: Mon, 19 Oct 2020 15:54:47 +0700 Subject: [PATCH 1/2] init v2.0 support mongodb --- Makefile | 20 +-- core/chaincode/chaincodetest.yaml | 2 +- .../kvledger/txmgmt/privacyenabledstate/db.go | 8 +- .../txmgmt/statedb/statecouchdb/couchdb.go | 34 ++-- .../txmgmt/statedb/statemongodb/mongodb.go | 145 +++++++++--------- .../statedb/statemongodb/mongodoc_conv.go | 31 ++-- docker-env.mk | 4 +- integration/nwo/core_template.go | 4 +- internal/peer/node/config.go | 38 +++-- sampleconfig/core.yaml | 2 +- 10 files changed, 156 insertions(+), 132 deletions(-) diff --git a/Makefile b/Makefile index 6f3343fd6c3..91cc618e574 100644 --- a/Makefile +++ b/Makefile @@ -222,16 +222,16 @@ $(BUILD_DIR)/images/peer/$(DUMMY): BUILD_ARGS=--build-arg GO_TAGS=${GO_TAGS} $(BUILD_DIR)/images/orderer/$(DUMMY): BUILD_ARGS=--build-arg GO_TAGS=${GO_TAGS} $(BUILD_DIR)/images/%/$(DUMMY): - @echo "Building Docker image $(DOCKER_NS)/fabric-$*" + @echo "Building Docker image $(DOCKER_NS_CUSTOM)/fabric-$*" @mkdir -p $(@D) $(DBUILD) -f images/$*/Dockerfile \ --build-arg GO_VER=$(GO_VER) \ --build-arg ALPINE_VER=$(ALPINE_VER) \ $(BUILD_ARGS) \ - -t $(DOCKER_NS)/fabric-$* ./$(BUILD_CONTEXT) - docker tag $(DOCKER_NS)/fabric-$* $(DOCKER_NS)/fabric-$*:$(BASE_VERSION) - docker tag $(DOCKER_NS)/fabric-$* $(DOCKER_NS)/fabric-$*:$(TWO_DIGIT_VERSION) - docker tag $(DOCKER_NS)/fabric-$* $(DOCKER_NS)/fabric-$*:$(DOCKER_TAG) + -t $(DOCKER_NS_CUSTOM)/fabric-$* ./$(BUILD_CONTEXT) + docker tag $(DOCKER_NS_CUSTOM)/fabric-$* $(DOCKER_NS_CUSTOM)/fabric-$*:$(BASE_VERSION) + docker tag $(DOCKER_NS_CUSTOM)/fabric-$* $(DOCKER_NS_CUSTOM)/fabric-$*:$(TWO_DIGIT_VERSION) + docker tag $(DOCKER_NS_CUSTOM)/fabric-$* $(DOCKER_NS_CUSTOM)/fabric-$*:$(DOCKER_TAG) @touch $@ # builds release packages for the host platform @@ -268,12 +268,12 @@ dist/%: release/% .PHONY: docker-list docker-list: $(RELEASE_IMAGES:%=%-docker-list) %-docker-list: - @echo $(DOCKER_NS)/fabric-$*:$(DOCKER_TAG) + @echo $(DOCKER_NS_CUSTOM)/fabric-$*:$(DOCKER_TAG) .PHONY: docker-clean docker-clean: $(RELEASE_IMAGES:%=%-docker-clean) %-docker-clean: - -@for image in "$$(docker images --quiet --filter=reference='$(DOCKER_NS)/fabric-$*:$(DOCKER_TAG)')"; do \ + -@for image in "$$(docker images --quiet --filter=reference='$(DOCKER_NS_CUSTOM)/fabric-$*:$(DOCKER_TAG)')"; do \ [ -z "$$image" ] || docker rmi -f $$image; \ done -@rm -rf $(BUILD_DIR)/images/$* || true @@ -281,18 +281,18 @@ docker-clean: $(RELEASE_IMAGES:%=%-docker-clean) .PHONY: docker-tag-latest docker-tag-latest: $(RELEASE_IMAGES:%=%-docker-tag-latest) %-docker-tag-latest: - docker tag $(DOCKER_NS)/fabric-$*:$(DOCKER_TAG) $(DOCKER_NS)/fabric-$*:latest + docker tag $(DOCKER_NS_CUSTOM)/fabric-$*:$(DOCKER_TAG) $(DOCKER_NS_CUSTOM)/fabric-$*:latest .PHONY: docker-tag-stable docker-tag-stable: $(RELEASE_IMAGES:%=%-docker-tag-stable) %-docker-tag-stable: - docker tag $(DOCKER_NS)/fabric-$*:$(DOCKER_TAG) $(DOCKER_NS)/fabric-$*:stable + docker tag $(DOCKER_NS_CUSTOM)/fabric-$*:$(DOCKER_TAG) $(DOCKER_NS_CUSTOM)/fabric-$*:stable .PHONY: publish-images publish-images: $(RELEASE_IMAGES:%=%-publish-images) %-publish-images: @docker login $(DOCKER_HUB_USERNAME) $(DOCKER_HUB_PASSWORD) - @docker push $(DOCKER_NS)/fabric-$*:$(PROJECT_VERSION) + @docker push $(DOCKER_NS_CUSTOM)/fabric-$*:$(PROJECT_VERSION) .PHONY: clean clean: docker-clean unit-test-clean release-clean diff --git a/core/chaincode/chaincodetest.yaml b/core/chaincode/chaincodetest.yaml index 5b3a7a1b6bc..ad71626dba0 100644 --- a/core/chaincode/chaincodetest.yaml +++ b/core/chaincode/chaincodetest.yaml @@ -304,7 +304,7 @@ chaincode: golang: # golang will never need more than baseos - runtime: $(DOCKER_NS)/fabric-baseos:$(TWO_DIGIT_VERSION) + runtime: hyperledger/fabric-baseos:$(TWO_DIGIT_VERSION) # whether or not golang chaincode should be linked dynamically dynamicLink: false diff --git a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go index 1a4f8390288..d227a8cfe74 100644 --- a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go +++ b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go @@ -20,8 +20,8 @@ import ( "github.com/hyperledger/fabric/core/ledger/kvledger/bookkeeping" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/statecouchdb" - "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/statemongodb" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/stateleveldb" + "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/statemongodb" "github.com/hyperledger/fabric/core/ledger/util" "github.com/pkg/errors" ) @@ -65,7 +65,8 @@ func NewDBProvider( var vdbProvider statedb.VersionedDBProvider var err error - logger.Debugf("constructing NewDBProvider stateDBConf %s",stateDBConf.MongoDB) + logger.Debugf("****StateDB****: %s", stateDBConf.StateDatabase) + //logger.Debugf("constructing NewDBProvider stateDBConf %s",stateDBConf.MongoDB) if stateDBConf != nil && stateDBConf.StateDatabase == couchDB { if vdbProvider, err = statecouchdb.NewVersionedDBProvider(stateDBConf.CouchDB, metricsProvider, sysNamespaces); err != nil { @@ -204,6 +205,7 @@ func (s *DB) GetPrivateDataHash(namespace, collection, key string) (*statedb.Ver // GetPrivateDataHash gets the value hash of a private data item identified by a tuple func (s *DB) GetValueHash(namespace, collection string, keyHash []byte) (*statedb.VersionedValue, error) { + logger.Debugf("GetValueHash") keyHashStr := string(keyHash) if !s.BytesKeySupported() { keyHashStr = base64.StdEncoding.EncodeToString(keyHash) @@ -263,7 +265,7 @@ func (s *DB) ApplyPrivacyAwareUpdates(updates *UpdateBatch, height *version.Heig addPvtUpdates(combinedUpdates, updates.PvtUpdates) addHashedUpdates(combinedUpdates, updates.HashUpdates, !s.BytesKeySupported()) s.metadataHint.setMetadataUsedFlag(updates) - logger.Debugf("ApplyPrivacyAwareUpdates %s",s.VersionedDB) + logger.Debugf("ApplyPrivacyAwareUpdates %s", s.VersionedDB) return s.VersionedDB.ApplyUpdates(combinedUpdates.UpdateBatch, height) } diff --git a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/couchdb.go b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/couchdb.go index 0c0c9a7c8f6..bb57c161a3b 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statecouchdb/couchdb.go +++ b/core/ledger/kvledger/txmgmt/statedb/statecouchdb/couchdb.go @@ -517,7 +517,7 @@ func (dbclient *couchDatabase) dropDatabase() (*dbOperationResponse, error) { func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc) (string, error) { dbName := dbclient.dbName - couchdbLogger.Debugf("[%s] Entering SaveDoc() id=[%s]", dbName, id) + couchdbLogger.Debugf("[%s] Entering SaveDoc() id=[%s] couchDoc: %+v", dbName, id, couchDoc) if !utf8.ValidString(id) { return "", errors.Errorf("doc id [%x] not a valid utf8 string", id) @@ -540,7 +540,7 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc //check to see if attachments is nil, if so, then this is a JSON only if couchDoc.attachments == nil { - + logger.Debugf("saveDoc() id: %s couchDoc.attachments == nil", id) //Test to see if this is a valid JSON if !isJSON(string(couchDoc.jsonValue)) { return "", errors.New("JSON format is not valid") @@ -548,8 +548,9 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc // if there are no attachments, then use the bytes passed in as the JSON data = couchDoc.jsonValue - + logger.Debugf("saveDoc() data: %+v", data) } else { // there are attachments + logger.Debugf("saveDoc() id: %s couchDoc.attachments != nil", id) //attachments are included, create the multipart definition multipartData, multipartBoundary, err3 := createAttachmentPart(couchDoc) @@ -566,7 +567,7 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc //Set the data buffer to the data from the create multi-part data data = multipartData.Bytes() - + logger.Debugf("saveDoc() data: %+v", data) //Set the default boundary to the value generated in the multipart creation defaultBoundary = multipartBoundary @@ -574,7 +575,7 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc //get the number of retries maxRetries := dbclient.couchInstance.conf.MaxRetries - + //couchdbLogger.Debugf("[SaveDoc() id:[%s] rev=[%s]",id, rev) //handle the request for saving document with a retry if there is a revision conflict resp, _, err := dbclient.handleRequestWithRevisionRetry(id, http.MethodPut, dbName, "SaveDoc", saveURL, data, rev, defaultBoundary, maxRetries, keepConnectionOpen, nil) @@ -589,8 +590,8 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc return "", err } - couchdbLogger.Debugf("[%s] Exiting SaveDoc()", dbclient.dbName) - + //couchdbLogger.Debugf("SaveDoc() %+v",revision) + couchdbLogger.Debugf("Database Name : [%s] Revision: [%s] Exiting saveDoc()", dbclient.dbName, revision) return revision, nil } @@ -599,13 +600,14 @@ func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc func (dbclient *couchDatabase) getDocumentRevision(id string) string { var rev = "" - + logger.Debugf("getDocumentRevision") //See if the document already exists, we need the rev for saves and deletes _, revdoc, err := dbclient.readDoc(id) if err == nil { //set the revision to the rev returned from the document read rev = revdoc } + logger.Debugf("getDocumentRevision rev: %v", rev) return rev } @@ -825,7 +827,7 @@ func (dbclient *couchDatabase) readDoc(id string) (*couchDoc, string, error) { } // for all multiparts couchDoc.attachments = attachments - + couchdbLogger.Debugf("return readDoc couchDoc: %+v, revision: %s, nil", couchDoc, revision) return &couchDoc, revision, nil } @@ -835,7 +837,7 @@ func (dbclient *couchDatabase) readDoc(id string) (*couchDoc, string, error) { return nil, "", errors.Wrap(err, "error reading response body") } - couchdbLogger.Debugf("[%s] Exiting ReadDoc()", dbclient.dbName) + couchdbLogger.Debugf("[%s] Exiting ReadDoc() id: %v revision: %v", dbclient.dbName, id, revision) return &couchDoc, revision, nil } @@ -1456,11 +1458,12 @@ func (dbclient *couchDatabase) batchRetrieveDocumentMetadata(keys []string) ([]* if err2 != nil { return nil, errors.Wrap(err2, "error unmarshalling json data") } - + couchdbLogger.Debugf("BatchRetrieveDocumentMetadata() jsonResponse:%+v", jsonResponse) docMetadataArray := []*docMetadata{} for _, row := range jsonResponse.Rows { docMetadata := &docMetadata{ID: row.ID, Rev: row.DocMetadata.Rev, Version: row.DocMetadata.Version} + couchdbLogger.Debugf("BatchRetrieveDocumentMetadata() docMetadata:%+v", docMetadata) docMetadataArray = append(docMetadataArray, docMetadata) } @@ -1473,7 +1476,7 @@ func (dbclient *couchDatabase) batchRetrieveDocumentMetadata(keys []string) ([]* //batchUpdateDocuments - batch method to batch update documents func (dbclient *couchDatabase) batchUpdateDocuments(documents []*couchDoc) ([]*batchUpdateResponse, error) { dbName := dbclient.dbName - + logger.Debugf("batchUpdateDocuments =[%v]", documents) if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { documentIdsString, err := printDocumentIds(documents) if err == nil { @@ -1494,7 +1497,7 @@ func (dbclient *couchDatabase) batchUpdateDocuments(documents []*couchDoc) ([]*b var jsonDocumentMap []interface{} for _, jsonDocument := range documents { - + couchdbLogger.Debugf("BatchUpdateDocuments() jsonDocument=[%+v]", jsonDocument) //create a document map var document = make(map[string]interface{}) @@ -1503,9 +1506,10 @@ func (dbclient *couchDatabase) batchUpdateDocuments(documents []*couchDoc) ([]*b if err != nil { return nil, errors.Wrap(err, "error unmarshalling json data") } - + couchdbLogger.Debugf("BatchUpdateDocuments() Unmarshal document: %+v", document) //iterate through any attachments if len(jsonDocument.attachments) > 0 { + couchdbLogger.Debugf("BatchUpdateDocuments() len(jsonDocument.attachments) > 0") //create a file attachment map fileAttachment := make(map[string]interface{}) @@ -1526,7 +1530,7 @@ func (dbclient *couchDatabase) batchUpdateDocuments(documents []*couchDoc) ([]*b jsonDocumentMap = append(jsonDocumentMap, document) } - + couchdbLogger.Debugf("BatchUpdateDocuments() jsonDocumentMap=[%+v]", jsonDocumentMap) //Add the documents to the "docs" item documentMap["docs"] = jsonDocumentMap diff --git a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go index 51143125818..a54f073131e 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go +++ b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go @@ -13,6 +13,7 @@ import ( "github.com/hyperledger/fabric/common/flogging" "github.com/hyperledger/fabric/core/ledger" "github.com/pkg/errors" + //"github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -40,7 +41,7 @@ type DBOperationResponse struct { //mongoDoc defines the structure for a JSON document value type mongoDoc struct { jsonValue []byte - binaryDatas []*binaryDataInfo + binaryDatas []*BinaryDataInfo } func (d *mongoDoc) key() (string, error) { @@ -51,10 +52,10 @@ func (d *mongoDoc) key() (string, error) { return m[idField].(string), nil } -type binaryDataInfo struct { +type BinaryDataInfo struct { Name string Length uint64 - BinaryData []byte + Binarydata []byte } //MongoConnectionDef contains parameters @@ -72,7 +73,7 @@ type MongoConnectionDef struct { //mongoInstance represents a MongoDB instance type mongoInstance struct { conf *ledger.MongoDBConfig //connection configuration - client *mongo.Client // a client to connect to this instance + client *mongo.Client // a client to connect to this instance stats *stats } @@ -101,32 +102,32 @@ type DBInfo struct { } // docMetadata is used for capturing MongoDB document header info, -// used to capture id, version, rev and binarydata returned in the query from MongoDB +// used to capture id, version, rev and binarydatas returned in the query from MongoDB type docMetadata struct { ID string `bson:"_id"` Rev int `bson:"_rev"` Version string `bson:"~version"` - binaryDatas []*binaryDataInfo `bson:"_binaryData"` + BinaryDatas []*BinaryDataInfo `bson:"_binaryData"` } type docMetadataJSON struct { - ID string `json:"_id"` - Version string `json:"~version"` + ID string `json:"_id"` + Version string `json:"~version"` } //batchUpdateResponse defines a structure for batch update response type batchUpdateResponse struct { - ID string `json:"id"` - Error string `json:"error"` - Reason string `json:"reason"` - Ok bool `json:"ok"` - Rev string `json:"rev"` + ID string `bson:"_id"` + Error string `bson:"error"` + Reason string `bson:"reason"` + Ok bool `bson:"ok"` + Rev int `bson:"_rev"` } type queryResult struct { id string `bson:"_id"` value []byte - binaryDatas []*binaryDataInfo `bson:"_binaryData"` + binaryDatas []*BinaryDataInfo `bson:"_binaryData"` } type RangeQueryResponse struct { @@ -159,12 +160,12 @@ type MongoQuery struct { } //attachmentInfo contains the definition for an attached file for mongodb -type attachmentInfo struct { - Name string - ContentType string `json:"content_type"` - Length uint64 - AttachmentBytes []byte `json:"data"` -} +//type attachmentInfo struct { +// Name string +// ContentType string `json:"content_type"` +// Length uint64 +// AttachmentBytes []byte `json:"data"` +//} //Paging info used for paging query //The result of query will not change with same query conditions(even updated data) @@ -221,6 +222,7 @@ func (mongoInstance *mongoInstance) healthCheck(ctx context.Context) error { } return nil } + // internalQueryLimit returns the maximum number of records to return internally // when querying MongoDB. func (mongoInstance *mongoInstance) queryLimit() int32 { @@ -286,7 +288,6 @@ func (dbclient *mongoDatabase) DropCollection() error { //from the database by id func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { var mongoDoc mongoDoc - var docMetadata *docMetadata var revision string dbName := dbclient.DatabaseName @@ -297,6 +298,7 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { client.Connect(ctx) logger.Debugf("Database Name : [%s] Collection Name : [%s] Entering readDoc() id=[%s]", dbName, colName, id) if !utf8.ValidString(id) { + logger.Error("doc id [%x] not a valid utf8 string", id) return nil, "", errors.Errorf("doc id [%x] not a valid utf8 string", id) } @@ -310,8 +312,10 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { return nil, "", res.Err() } - - err := res.Decode(&docMetadata) + var doc = bson.M{} + _ = res.Decode(&doc) + var docMetadata = &docMetadata{} + err := res.Decode(docMetadata) if err != nil { if strings.Contains(err.Error(), "no documents in result") { logger.Debugf("Database Name : [%s] Collection Name : [%s] Document not found", dbclient.DatabaseName, dbclient.CollectionName) @@ -326,13 +330,13 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { return nil, "", res.Err() } - if docMetadata.binaryDatas != nil { - for _, binaryDataInfoVal := range docMetadata.binaryDatas { - name := binaryDataInfoVal.Name - length := binaryDataInfoVal.Length - binaryData := binaryDataInfoVal.BinaryData - binaryDataInfoVal := binaryDataInfo{Name: name, Length: length, BinaryData: binaryData} - mongoDoc.binaryDatas = append(mongoDoc.binaryDatas, &binaryDataInfoVal) + if docMetadata.BinaryDatas != nil { + for _, BinaryDataInfoVal := range docMetadata.BinaryDatas { + name := BinaryDataInfoVal.Name + length := BinaryDataInfoVal.Length + binaryData := BinaryDataInfoVal.Binarydata + BinaryDataInfoVal := BinaryDataInfo{Name: name, Length: length, Binarydata: binaryData} + mongoDoc.binaryDatas = append(mongoDoc.binaryDatas, &BinaryDataInfoVal) } } @@ -364,24 +368,24 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { data, err := json.Marshal(jsonValue) if err != nil { + logger.Errorf("error marshalling json data") return nil, "", errors.Wrap(err, "error marshalling json data") } mongoDoc.jsonValue = data - logger.Debugf("Database Name : [%s] Collection Name : [%s] Exiting readDoc()", dbName, colName) return &mongoDoc, revision, nil } -func encodeForJSON(str string) (string, error) { - buf := &bytes.Buffer{} - encoder := json.NewEncoder(buf) - if err := encoder.Encode(str); err != nil { - return "", errors.Wrap(err, "error encoding json data") - } - // Encode adds double quotes to string and terminates with \n - stripping them as bytes as they are all ascii(0-127) - buffer := buf.Bytes() - return string(buffer[1 : len(buffer)-2]), nil -} +//func encodeForJSON(str string) (string, error) { +// buf := &bytes.Buffer{} +// encoder := json.NewEncoder(buf) +// if err := encoder.Encode(str); err != nil { +// return "", errors.Wrap(err, "error encoding json data") +// } +// // Encode adds double quotes to string and terminates with \n - stripping them as bytes as they are all ascii(0-127) +// buffer := buf.Bytes() +// return string(buffer[1 : len(buffer)-2]), nil +//} func encodePathElement(str string) string { @@ -457,7 +461,6 @@ func (dbclient *mongoDatabase) saveDoc(id string, rev string, mongoDoc *mongoDoc dbName := dbclient.DatabaseName colName := dbclient.CollectionName defer dbclient.mongoInstance.recordMetric(time.Now(), colName, "saveDoc") - logger.Debugf("Database Name : [%s] Collection Name : [%s] Entering saveDoc() id=[%s]", dbName, colName, id) if !utf8.ValidString(id) { return "", errors.Errorf("doc id [%x] not a valid utf8 string", id) @@ -482,7 +485,6 @@ func (dbclient *mongoDatabase) saveDoc(id string, rev string, mongoDoc *mongoDoc } } else { // there are binaryDatas - binaryDataJSONMap[binaryField] = mongoDoc.binaryDatas if mongoDoc.jsonValue != nil { @@ -497,7 +499,6 @@ func (dbclient *mongoDatabase) saveDoc(id string, rev string, mongoDoc *mongoDoc binaryDataJSONMap[jsonKey] = jsonValue } } - } client := dbclient.mongoInstance.client @@ -568,8 +569,11 @@ func (dbclient *mongoDatabase) batchRetrieveDocumentMetadata(keys []string) ([]* //keymap := make(map[string]interface{}) //keymap["keys"] = keys - cursor, _ := client.Database(dbName).Collection(colName, nil).Find(ctx, bson.D{{idField, bson.D{{"$in", keys}}}}) - + cursor, err := client.Database(dbName).Collection(colName, nil).Find(ctx, bson.D{{idField, bson.D{{"$in", keys}}}}) + if err != nil { + logger.Errorf("batchRetrieveDocumentMetadata() Find() err: %s", err.Error()) + return nil, err + } docMetadataArray := []*docMetadata{} //for _, row := range jsonResponse.Rows { @@ -577,13 +581,13 @@ func (dbclient *mongoDatabase) batchRetrieveDocumentMetadata(keys []string) ([]* // docMetadataArray = append(docMetadataArray, docMetadata) //} - for cursor.Next(context.TODO()) { + for cursor.Next(ctx) { var docMetadata = &docMetadata{} err := cursor.Decode(docMetadata) if err != nil { - + logger.Errorf("batchRetrieveDocumentMetadata() err: %s", err.Error()) + return nil, err } - docMetadataArray = append(docMetadataArray, docMetadata) } @@ -595,7 +599,6 @@ func (dbclient *mongoDatabase) batchRetrieveDocumentMetadata(keys []string) ([]* //batchUpdateDocuments - batch method to batch update documents func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*batchUpdateResponse, error) { - logger.Debugf("batchUpdateDocuments =[%v]", documents) dbName := dbclient.DatabaseName colName := dbclient.CollectionName client := dbclient.mongoInstance.client @@ -603,7 +606,7 @@ func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*b client.Connect(ctx) if logger.IsEnabledFor(zapcore.DebugLevel) { - + documentIdsString, err := printDocumentIds(documents) if err == nil { logger.Debugf("Database Name : [%s] Collection Name : [%s] Entering batchUpdateDocuments() document ids=[%s]", dbName, colName, documentIdsString) @@ -625,7 +628,6 @@ func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*b if err != nil { return nil, errors.Wrap(err, "error unmarshalling json data") } - id := document[idField] delete(document, revField) deleted := document[deletedField] @@ -651,7 +653,7 @@ func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*b } - var response []*batchUpdateResponse + var response = []*batchUpdateResponse{} var raw []string for _, sr := range resultMap { var value = &batchUpdateResponse{} @@ -665,7 +667,6 @@ func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*b str := fmt.Sprintf("ID : [%s] Rev : [%d] Error : [%s] Reason : [%s]", value.ID, value.Rev, value.Error, value.Reason) raw = append(raw, str) } - logger.Debugf("Database Name : [%s] Collection Name : [%s] Exiting batchUpdateDocuments() response=[\n %s \n]", dbName, colName, strings.Join(raw, "\n")) return response, nil @@ -673,7 +674,6 @@ func (dbclient *mongoDatabase) batchUpdateDocuments(documents []*mongoDoc) ([]*b //getDocumentRevision will return the revision if the document exists, otherwise it will return "" func (dbclient *mongoDatabase) getDocumentRevision(id string) string { - var rev = "" //See if the document already exists, we need the rev for saves and deletes @@ -694,11 +694,7 @@ func printDocumentIds(documentPointers []*mongoDoc) (string, error) { for _, documentPointer := range documentPointers { docMetadataTemp := &docMetadataJSON{} err := json.Unmarshal(documentPointer.jsonValue, &docMetadataTemp) - // err := documentPointer.jsonValue.Decode(&docMetadataTemp) - logger.Debugf("docMetadata jsonValue =[%s]", documentPointer.jsonValue) - logger.Debugf("docMetadata =[%v]", docMetadataTemp) if err != nil { - logger.Debugf("docMetadata.ID =[%s]", docMetadataTemp.ID) return "", errors.Wrap(err, "error unmarshalling json data") } documentIds = append(documentIds, docMetadataTemp.ID) @@ -799,8 +795,8 @@ func (dbclient *mongoDatabase) readDocRange(startKey, endKey string, limit int32 if err != nil { return nil, "", errors.Wrap(err, "error marshalling json data") } - var binaryDatas []*binaryDataInfo - for _, binaryData := range docMetaData.binaryDatas { + var binaryDatas []*BinaryDataInfo + for _, binaryData := range docMetaData.BinaryDatas { binaryDatas = append(binaryDatas, binaryData) } @@ -928,7 +924,6 @@ func (dbclient *mongoDatabase) DropIndex(indexName string) error { return nil } - //warmIndex method provides a function for warming a single index // func (dbclient *mongoDatabase) warmIndex(designdoc, indexname string) error { // dbName := dbclient.dbName @@ -1058,13 +1053,11 @@ func (dbclient *mongoDatabase) queryDocuments(query string) ([]*queryResult, str err := resultCur.Decode(&docMetadata) if err != nil { - if err != nil { - if strings.Contains(err.Error(), "no documents in result") { - logger.Debugf("Database Name : [%s] Collection Name : [%s] Document not found", dbclient.DatabaseName, dbclient.CollectionName) - return nil, "", nil - } - return nil, "", err + if strings.Contains(err.Error(), "no documents in result") { + logger.Debugf("Database Name : [%s] Collection Name : [%s] Document not found", dbclient.DatabaseName, dbclient.CollectionName) + return nil, "", nil } + return nil, "", err } var docMap map[string]interface{} @@ -1074,8 +1067,8 @@ func (dbclient *mongoDatabase) queryDocuments(query string) ([]*queryResult, str } resultValue, _ := json.Marshal(docMap) - var binaryDatas []*binaryDataInfo - for _, binaryData := range docMetadata.binaryDatas { + var binaryDatas []*BinaryDataInfo + for _, binaryData := range docMetadata.BinaryDatas { binaryDatas = append(binaryDatas, binaryData) } @@ -1142,7 +1135,7 @@ func (mongoInstance *mongoInstance) verifyMongoConfig() error { //get the number of retries for startup maxRetriesOnStartup := mongoInstance.conf.MaxRetriesOnStartup - ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 35*time.Second) for attempts := 0; attempts <= maxRetriesOnStartup; attempts++ { _ = mongoInstance.client.Connect(ctx) err := mongoInstance.client.Ping(ctx, nil) @@ -1154,11 +1147,15 @@ func (mongoInstance *mongoInstance) verifyMongoConfig() error { //backoff, doubling the retry time for next attempt waitDuration *= 2 + } else { + logger.Debugf("verifyMongoConfig() Done") + return nil } } return nil } + // isEmpty returns false if mongoInstance contains any databases // (except mongodb system databases and any database name supplied in the parameter 'databasesToIgnore') func (mongoInstance *mongoInstance) isEmpty(databasesToIgnore []string) (bool, error) { @@ -1182,16 +1179,16 @@ func (mongoInstance *mongoInstance) isEmpty(databasesToIgnore []string) (bool, e // retrieveApplicationDBNames returns all the application database names in the mongo instance func (mongoInstance *mongoInstance) retrieveApplicationDBNames() ([]string, error) { - + logger.Debugf("Entering retrieveApplicationDBNames()") defer logger.Debugf("Exiting retrieveApplicationDBNames()") - + client := mongoInstance.client ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) _ = client.Connect(ctx) filter := bson.D{{}} - dbNames,err := client.ListDatabaseNames(ctx,filter) + dbNames, err := client.ListDatabaseNames(ctx, filter) if err != nil { return nil, errors.WithMessage(err, "can't get list Database name") } diff --git a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodoc_conv.go b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodoc_conv.go index eea5e0ec541..c146552aa5f 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodoc_conv.go +++ b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodoc_conv.go @@ -10,9 +10,9 @@ import ( "bytes" "encoding/json" "fmt" + "regexp" "strings" "unicode/utf8" - "regexp" "github.com/hyperledger/fabric/core/ledger/internal/version" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" @@ -20,7 +20,7 @@ import ( ) const ( - binaryField = "_binaryData" + binaryField = "_binaryData" binaryWrapper = "valueBytes" idField = "_id" revField = "_rev" @@ -101,7 +101,6 @@ func validateAndRetrieveFields(doc *mongoDoc) (*mongoDocFields, error) { if err := decoder.Decode(&jsonDoc); err != nil { return nil, err } - docFields := &mongoDocFields{} docFields.id = jsonDoc[idField].(string) if jsonDoc[revField] != nil { @@ -111,7 +110,6 @@ func validateAndRetrieveFields(doc *mongoDoc) (*mongoDocFields, error) { return nil, fmt.Errorf("version field %s was not found", versionField) } docFields.versionAndMetadata = jsonDoc[versionField].(string) - delete(jsonDoc, idField) delete(jsonDoc, revField) delete(jsonDoc, versionField) @@ -123,9 +121,24 @@ func validateAndRetrieveFields(doc *mongoDoc) (*mongoDocFields, error) { } for _, binaryData := range doc.binaryDatas { if binaryData.Name == binaryWrapper { - docFields.value = binaryData.BinaryData + docFields.value = binaryData.Binarydata + } + } + // handle binary or json data + if doc.binaryDatas != nil { // binary attachment + // get binary data from attachment + for _, attachment := range doc.binaryDatas { + if attachment.Name == binaryField { + docFields.value = attachment.Binarydata + } + } + } else { + // marshal the returned JSON data. + if docFields.value, err = json.Marshal(jsonDoc); err != nil { + return nil, err } } + logger.Debugf("validateAndRetrieveFields docFields after : %+v", docFields) return docFields, err } @@ -181,10 +194,10 @@ func keyValToMongoDoc(kv *keyValue) (*mongoDoc, error) { mongoDoc := &mongoDoc{jsonValue: jsonBytes} if kvtype == kvTypeBinaryData { - binaryData := &binaryDataInfo{} - binaryData.BinaryData = value + binaryData := &BinaryDataInfo{} + binaryData.Binarydata = value binaryData.Name = binaryField - binaryDatas := append([]*binaryDataInfo{}, binaryData) + binaryDatas := append([]*BinaryDataInfo{}, binaryData) mongoDoc.binaryDatas = binaryDatas } return mongoDoc, nil @@ -277,7 +290,7 @@ func decodeDataformatInfo(mongoDoc *mongoDoc) (string, error) { logger.Errorf("%+v", err) return "", err } - version := regexp.MustCompile(`^"(.*)"$`).ReplaceAllString(dataformatInfo.Version ,`$1`) + version := regexp.MustCompile(`^"(.*)"$`).ReplaceAllString(dataformatInfo.Version, `$1`) return version, nil } diff --git a/docker-env.mk b/docker-env.mk index 0d5036d6eb2..044f3a36d3d 100644 --- a/docker-env.mk +++ b/docker-env.mk @@ -23,8 +23,8 @@ endif DBUILD = docker build --force-rm $(DOCKER_BUILD_FLAGS) -DOCKER_NS ?= koankem0901 -DOCKER_NS_HYPERLEDGER_FABRIC ?= hyperledger +DOCKER_NS_CUSTOM ?= koankem0901 +DOCKER_NS ?= hyperledger DOCKER_TAG=$(ARCH)-$(PROJECT_VERSION) BASE_DOCKER_LABEL=org.akachain.fabric diff --git a/integration/nwo/core_template.go b/integration/nwo/core_template.go index 9490f824304..2d4f1f60eff 100644 --- a/integration/nwo/core_template.go +++ b/integration/nwo/core_template.go @@ -167,10 +167,10 @@ chaincode: builder: $(DOCKER_NS)/fabric-ccenv:$(PROJECT_VERSION) pull: false golang: - runtime: $(DOCKER_NS)/fabric-baseos:$(PROJECT_VERSION) + runtime: hyperledger/fabric-baseos:$(PROJECT_VERSION) dynamicLink: false car: - runtime: $(DOCKER_NS)/fabric-baseos:$(PROJECT_VERSION) + runtime: hyperledger/fabric-baseos:$(PROJECT_VERSION) java: runtime: $(DOCKER_NS)/fabric-javaenv:latest node: diff --git a/internal/peer/node/config.go b/internal/peer/node/config.go index 1b0a78cb568..c638a04e5d9 100644 --- a/internal/peer/node/config.go +++ b/internal/peer/node/config.go @@ -7,18 +7,22 @@ SPDX-License-Identifier: Apache-2.0 package node import ( - "path/filepath" - "regexp" - "strings" + "github.com/hyperledger/fabric/common/flogging" coreconfig "github.com/hyperledger/fabric/core/config" "github.com/hyperledger/fabric/core/ledger" "github.com/spf13/viper" + "path/filepath" + "regexp" + "strings" ) + const ( - couchDB = "couchdb" - mongoDB = "mongodb" + couchDB = "couchdb" + mongoDB = "mongodb" ) +var configLogger = flogging.MustGetLogger("configLogger") + // escapeUpperCase replaces every upper case letter with a '$' and the respective // lower-case letter func escapeUpperCase(stateDatabaseName string) string { @@ -35,7 +39,7 @@ func ledgerConfig() *ledger.Config { maxBatchUpdateSize := 500 collElgProcMaxDbBatchSize := 5000 collElgProcDbBatchesInterval := 1000 - if couchDB == stateDB{ + if couchDB == stateDB { if viper.IsSet("ledger.state.couchDBConfig.warmIndexesAfterNBlocks") { warmAfterNBlocks = viper.GetInt("ledger.state.couchDBConfig.warmIndexesAfterNBlocks") } @@ -45,7 +49,7 @@ func ledgerConfig() *ledger.Config { if viper.IsSet("ledger.state.couchDBConfig.maxBatchUpdateSize") { maxBatchUpdateSize = viper.GetInt("ledger.state.couchDBConfig.maxBatchUpdateSize") } - }else if mongoDB == stateDB{ + } else if mongoDB == stateDB { if viper.IsSet("ledger.state.mongoDBConfig.warmIndexesAfterNBlocks") { warmAfterNBlocks = viper.GetInt("ledger.state.mongoDBConfig.warmIndexesAfterNBlocks") } @@ -103,19 +107,23 @@ func ledgerConfig() *ledger.Config { } } else if conf.StateDBConfig.StateDatabase == mongoDB { conf.StateDBConfig.MongoDB = &ledger.MongoDBConfig{ - Address: viper.GetString("ledger.state.mongoDBConfig.mongoDBAddress"), - Username: viper.GetString("ledger.state.mongoDBConfig.username"), - DatabaseName: "statedb", - Password: viper.GetString("ledger.state.mongoDBConfig.password"), - MaxRetries: viper.GetInt("ledger.state.mongoDBConfig.maxRetries"), - MaxRetriesOnStartup: viper.GetInt("ledger.state.mongoDBConfig.maxRetriesOnStartup"), - RequestTimeout: viper.GetDuration("ledger.state.mongoDBConfig.requestTimeout"), - QueryLimit: internalQueryLimit, + Address: viper.GetString("ledger.state.mongoDBConfig.mongoDBAddress"), + Username: viper.GetString("ledger.state.mongoDBConfig.username"), + DatabaseName: "statemongodb", + Password: viper.GetString("ledger.state.mongoDBConfig.password"), + //MaxRetries: viper.GetInt("ledger.state.mongoDBConfig.maxRetries"), + //MaxRetriesOnStartup: viper.GetInt("ledger.state.mongoDBConfig.maxRetriesOnStartup"), + //RequestTimeout: viper.GetDuration("ledger.state.mongoDBConfig.requestTimeout"), + MaxRetries: 3, + MaxRetriesOnStartup: 3, + RequestTimeout: 35000000000, + QueryLimit: internalQueryLimit, MaxBatchUpdateSize: maxBatchUpdateSize, WarmIndexesAfterNBlocks: warmAfterNBlocks, RedoLogPath: filepath.Join(rootFSPath, "mongoRedoLogs"), UserCacheSizeMBs: viper.GetInt("ledger.state.mongoDBConfig.cacheSize"), } + configLogger.Debugf("conf.StateDBConfig.MongoDB %v", conf.StateDBConfig.MongoDB) } return conf } diff --git a/sampleconfig/core.yaml b/sampleconfig/core.yaml index 56c8193db64..98af20da472 100644 --- a/sampleconfig/core.yaml +++ b/sampleconfig/core.yaml @@ -527,7 +527,7 @@ chaincode: golang: # golang will never need more than baseos - runtime: $(DOCKER_NS)/fabric-baseos:$(TWO_DIGIT_VERSION) + runtime: hyperledger/fabric-baseos:$(TWO_DIGIT_VERSION) # whether or not golang chaincode should be linked dynamically dynamicLink: false From 092bac78ec2367ca92a6fc98debc13d75f062b5c Mon Sep 17 00:00:00 2001 From: thienlk1 Date: Wed, 18 Nov 2020 09:04:18 +0700 Subject: [PATCH 2/2] fix validator index format --- .gitignore | 2 +- .../statedb/statemongodb/commit_handling.go | 1 - .../txmgmt/statedb/statemongodb/mongodb.go | 10 ++- internal/ccmetadata/validators.go | 39 +++++++-- internal/ccmetadata/validators_test.go | 79 +++++++++++++++++++ internal/peer/node/config.go | 2 +- 6 files changed, 122 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 168f24058f3..981ce9b9935 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ tags TESTS*.xml .tox/ .vagrant/ -.vscode +.vscode \ No newline at end of file diff --git a/core/ledger/kvledger/txmgmt/statedb/statemongodb/commit_handling.go b/core/ledger/kvledger/txmgmt/statedb/statemongodb/commit_handling.go index 947b94044fd..2b05bd1aa51 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statemongodb/commit_handling.go +++ b/core/ledger/kvledger/txmgmt/statedb/statemongodb/commit_handling.go @@ -186,7 +186,6 @@ func (c *committer) commitUpdates() error { // IF INDIVIDUAL DOCUMENTS IN THE BULK UPDATE DID NOT SUCCEED, TRY THEM INDIVIDUALLY // iterate through the response from MongoDB by document for _, resp := range responses { - logger.Debugf("commitUpdates resp.Error :%s", resp.Error) // If the document returned an error, retry the individual document if resp.Error != "" || resp.ID == "" { rev := string(resp.Rev) diff --git a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go index a54f073131e..48632685b25 100644 --- a/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go +++ b/core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go @@ -55,7 +55,7 @@ func (d *mongoDoc) key() (string, error) { type BinaryDataInfo struct { Name string Length uint64 - Binarydata []byte + Binarydata []byte `bson:"Binarydata"` } //MongoConnectionDef contains parameters @@ -357,7 +357,11 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { jsonValue[key] = int64(value.Value().Int32()) } } else { - jsonValue[key] = value.Value().String() + if _, ok := value.Value().StringValueOK(); ok { + jsonValue[key] = value.Value().StringValue() + } else { + jsonValue[key] = value.Value().String() + } } } @@ -373,6 +377,8 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) { } mongoDoc.jsonValue = data logger.Debugf("Database Name : [%s] Collection Name : [%s] Exiting readDoc()", dbName, colName) + logger.Debugf("jsonValue : %+v", jsonValue) + logger.Debugf("mongoDoc : %s", string(mongoDoc.jsonValue)) return &mongoDoc, revision, nil } diff --git a/internal/ccmetadata/validators.go b/internal/ccmetadata/validators.go index 2fc035fef3d..c6f813a2246 100644 --- a/internal/ccmetadata/validators.go +++ b/internal/ccmetadata/validators.go @@ -25,17 +25,19 @@ type fileValidator func(fileName string, fileBytes []byte) error // AllowedCharsCollectionName captures the regex pattern for a valid collection name const AllowedCharsCollectionName = "[A-Za-z0-9_-]+" -// Currently, the only metadata expected and allowed is for META-INF/statedb/couchdb/indexes. +// Currently, the only metadata expected and allowed is for META-INF/statedb/[couchdb, mongodb]/indexes. var fileValidators = map[*regexp.Regexp]fileValidator{ regexp.MustCompile("^META-INF/statedb/couchdb/indexes/.*[.]json"): couchdbIndexFileValidator, regexp.MustCompile("^META-INF/statedb/couchdb/collections/" + AllowedCharsCollectionName + "/indexes/.*[.]json"): couchdbIndexFileValidator, + regexp.MustCompile("^META-INF/statedb/mongodb/indexes/.*[.]json"): mongodbIndexFileValidator, + regexp.MustCompile("^META-INF/statedb/mongodb/collections/" + AllowedCharsCollectionName + "/indexes/.*[.]json"): mongodbIndexFileValidator, } var collectionNameValid = regexp.MustCompile("^" + AllowedCharsCollectionName) var fileNameValid = regexp.MustCompile("^.*[.]json") -var validDatabases = []string{"couchdb"} +var validDatabases = []string{"couchdb", "mongodb"} // UnhandledDirectoryError is returned for metadata files in unhandled directories type UnhandledDirectoryError struct { @@ -77,7 +79,7 @@ func ValidateMetadataFile(filePathName string, fileBytes []byte) error { } func buildMetadataFileErrorMessage(filePathName string) string { - + logger.Debugf("buildMetadataFileErrorMessage") dir, filename := filepath.Split(filePathName) if !strings.HasPrefix(filePathName, "META-INF/statedb") { @@ -89,8 +91,9 @@ func buildMetadataFileErrorMessage(filePathName string) string { return fmt.Sprintf("metadata file path must include a database and index directory: %s", dir) } // validate the database type + logger.Debugf("validDatabases: %s", validDatabases) if !contains(validDatabases, directoryArray[2]) { - return fmt.Sprintf("database name [%s] is not supported, valid options: %s", directoryArray[2], validDatabases) + return fmt.Sprintf("DATABASE name [%s] is not supported, valid options: %s", directoryArray[2], validDatabases) } // verify "indexes" is under the database name if len(directoryArray) == 4 && directoryArray[3] != "indexes" { @@ -156,6 +159,24 @@ func couchdbIndexFileValidator(fileName string, fileBytes []byte) error { } +// mongodbIndexFileValidator implements fileValidator +func mongodbIndexFileValidator(fileName string, fileBytes []byte) error { + // if the content does not validate as JSON, return err to invalidate the file + boolIsJSON, indexDefinition := isJSON(fileBytes) + if !boolIsJSON { + return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", fileName)} + } + + //validate the index definition + err := validateIndexJSON(indexDefinition) + if err != nil { + return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", fileName, err)} + } + + return nil + +} + // isJSON tests a string to determine if it can be parsed as valid JSON func isJSON(s []byte) (bool, map[string]interface{}) { var js map[string]interface{} @@ -297,10 +318,16 @@ func validateFieldMap(jsonFragment map[string]interface{}) error { //Ensure the sort is either "asc" or "desc" jv := strings.ToLower(jsonValue) if jv != "asc" && jv != "desc" { - return fmt.Errorf("Sort must be either \"asc\" or \"desc\". \"%s\" was found.", jsonValue) + return fmt.Errorf("Sort must be either \"asc\",\"desc\",1 or -1. \"%s\" was found.", jsonValue) + } + logger.Debugf("Found index field name: \"%s\":\"%s\"", jsonKey, jsonValue) + case float64: + //Ensure the sort is either 1 or -1 + jv := int(jsonValue) + if jv != 1 && jv != -1 { + return fmt.Errorf("Sort must be either \"asc\",\"desc\",1 or -1. \"%d\" was found.", jsonValue) } logger.Debugf("Found index field name: \"%s\":\"%s\"", jsonKey, jsonValue) - default: return fmt.Errorf("Invalid field definition, fields must be in the form \"fieldname\":\"sort\"") } diff --git a/internal/ccmetadata/validators_test.go b/internal/ccmetadata/validators_test.go index 51ca4151b03..ed03a6af248 100644 --- a/internal/ccmetadata/validators_test.go +++ b/internal/ccmetadata/validators_test.go @@ -27,6 +27,12 @@ func TestGoodIndexJSON(t *testing.T) { err := ValidateMetadataFile(fileName, fileBytes) assert.NoError(t, err, "Error validating a good index") + + fileName = "META-INF/statedb/mongodb/indexes/myIndex.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + assert.NoError(t, err, "Error validating a good index") } func TestBadIndexJSON(t *testing.T) { @@ -46,6 +52,19 @@ func TestBadIndexJSON(t *testing.T) { assert.True(t, ok, "Should have received an InvalidIndexContentError") t.Log("SAMPLE ERROR STRING:", err.Error()) + + fileName = "META-INF/statedb/mongodb/indexes/myIndex.json" + fileBytes = []byte("invalid json") + + err = ValidateMetadataFile(fileName, fileBytes) + + assert.Error(t, err, "Should have received an InvalidIndexContentError") + + // Type assertion on InvalidIndexContentError + _, ok = err.(*InvalidIndexContentError) + assert.True(t, ok, "Should have received an InvalidIndexContentError") + + t.Log("SAMPLE ERROR STRING:", err.Error()) } func TestIndexWrongLocation(t *testing.T) { @@ -64,6 +83,18 @@ func TestIndexWrongLocation(t *testing.T) { assert.True(t, ok, "Should have received an UnhandledDirectoryError") t.Log("SAMPLE ERROR STRING:", err.Error()) + + fileName = "META-INF/statedb/mongodb/myIndex.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner"`) + + err = ValidateMetadataFile(fileName, fileBytes) + assert.Error(t, err, "Should have received an UnhandledDirectoryError") + + // Type assertion on UnhandledDirectoryError + _, ok = err.(*UnhandledDirectoryError) + assert.True(t, ok, "Should have received an UnhandledDirectoryError") + + t.Log("SAMPLE ERROR STRING:", err.Error()) } func TestInvalidMetadataType(t *testing.T) { @@ -80,6 +111,7 @@ func TestInvalidMetadataType(t *testing.T) { // Type assertion on UnhandledDirectoryError _, ok := err.(*UnhandledDirectoryError) assert.True(t, ok, "Should have received an UnhandledDirectoryError") + } func TestBadMetadataExtension(t *testing.T) { @@ -108,6 +140,14 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.Error(t, err, "Should have received an error for bad META-INF directory") + // Test bad META-INF + fileName = "META-INF1/statedb/mongodb/indexes/test1.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.Error(t, err, "Should have received an error for bad META-INF directory") + // Test bad path length fileName = "META-INF/statedb/test1.json" fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) @@ -116,6 +156,14 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.Error(t, err, "Should have received an error for bad length") + // Test invalid indexes directory name + fileName = "META-INF/statedb/mongodb/index/test1.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.Error(t, err, "Should have received an error for invalid indexes directory") + // Test invalid database name fileName = "META-INF/statedb/goleveldb/indexes/test1.json" fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) @@ -140,6 +188,14 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.Error(t, err, "Should have received an error for invalid collections directory") + // Test invalid collections directory name + fileName = "META-INF/statedb/mongodb/collection/testcoll/indexes/test1.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.Error(t, err, "Should have received an error for invalid collections directory") + // Test valid collections name fileName = "META-INF/statedb/couchdb/collections/testcoll/indexes/test1.json" fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) @@ -148,6 +204,14 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.NoError(t, err, "Error should not have been thrown for a valid collection name") + // Test valid collections name + fileName = "META-INF/statedb/mongodb/collections/testcoll/indexes/test1.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.NoError(t, err, "Error should not have been thrown for a valid collection name") + // Test invalid collections name fileName = "META-INF/statedb/couchdb/collections/#testcoll/indexes/test1.json" fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) @@ -156,6 +220,14 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.Error(t, err, "Should have received an error for an invalid collection name") + // Test invalid collections name + fileName = "META-INF/statedb/mongodb/collections/#testcoll/indexes/test1.json" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.Error(t, err, "Should have received an error for an invalid collection name") + // Test invalid collections name fileName = "META-INF/statedb/couchdb/collections/testcoll/indexes/test1.txt" fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) @@ -164,6 +236,13 @@ func TestBadFilePaths(t *testing.T) { fmt.Println(err) assert.Error(t, err, "Should have received an error for an invalid file name") + // Test invalid collections name + fileName = "META-INF/statedb/mongodb/collections/testcoll/indexes/test1.txt" + fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`) + + err = ValidateMetadataFile(fileName, fileBytes) + fmt.Println(err) + assert.Error(t, err, "Should have received an error for an invalid file name") } func TestIndexValidation(t *testing.T) { diff --git a/internal/peer/node/config.go b/internal/peer/node/config.go index c638a04e5d9..3d7aa33736d 100644 --- a/internal/peer/node/config.go +++ b/internal/peer/node/config.go @@ -116,7 +116,7 @@ func ledgerConfig() *ledger.Config { //RequestTimeout: viper.GetDuration("ledger.state.mongoDBConfig.requestTimeout"), MaxRetries: 3, MaxRetriesOnStartup: 3, - RequestTimeout: 35000000000, + RequestTimeout: 350000000000, QueryLimit: internalQueryLimit, MaxBatchUpdateSize: maxBatchUpdateSize, WarmIndexesAfterNBlocks: warmAfterNBlocks,