Skip to content

Commit

Permalink
FAB-2531 Range queries fail iterating beyond 100 items
Browse files Browse the repository at this point in the history 
This is a temporary stabilization change until the following issue is
resolved and will prevent errors in queries for the beta:
FAB-2462 - Re-introduce paging for range queries and rich queries

Peer sends 100 results to shim.
When shim asks for 101st item, the request for the next 100 goes
back to peer and it fails due to a closed iterator.

This change also sets the HasMore flag on query result sets to
false.  This will prevent the requery from being called by the
shim and throwing an exception.

Purpose of this change is to stabilize the behavior for the
beta by setting a query size limit in core.yaml at 10000.

Then use the query size limit in the CouchDB query and in
the chaincode handler.

Added changes to chaincodetest.yaml to match core.yaml.

Added unit test cases for the new querylimit option.

Change-Id: I772d1f87beec2296db2eed68a0528181ac1ddeca
Signed-off-by: Chris Elder <chris.elder@us.ibm.com>
  • Loading branch information
Chris Elder committed Mar 7, 2017
1 parent ec1bbdf commit 0fc6c4d
Show file tree
Hide file tree
Showing 10 changed files with 402 additions and 98 deletions.
54 changes: 16 additions & 38 deletions core/chaincode/chaincodetest.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,7 @@ chaincode:
lccc: enable
escc: enable
vscc: enable

###############################################################################
#
# Ledger section - ledger configuration encompases both the blockchain
Expand All @@ -402,45 +403,22 @@ ledger:

blockchain:

# Setting the deploy-system-chaincode property to false will prevent the
# deploying of system chaincode at genesis time.
deploy-system-chaincode: false

state:

# Control the number state deltas that are maintained. This takes additional
# disk space, but allow the state to be rolled backwards and forwards
# without the need to replay transactions.
deltaHistorySize: 500

# The data structure in which the state will be stored. Different data
# structures may offer different performance characteristics.
# Options are 'buckettree', 'trie' and 'raw'.
# ( Note:'raw' is experimental and incomplete. )
# If not set, the default data structure is the 'buckettree'.
# This CANNOT be changed after the DB has been created.
dataStructure:
# The name of the data structure is for storing the state
name: buckettree
# The data structure specific configurations
configs:
# configurations for 'bucketree'. These CANNOT be changed after the DB
# has been created. 'numBuckets' defines the number of bins that the
# state key-values are to be divided
numBuckets: 1000003
# 'maxGroupingAtEachLevel' defines the number of bins that are grouped
#together to construct next level of the merkle-tree (this is applied
# repeatedly for constructing the entire tree).
maxGroupingAtEachLevel: 5
# 'bucketCacheSize' defines the size (in MBs) of the cache that is used to keep
# the buckets (from root upto secondlast level) in memory. This cache helps
# in making state hash computation faster. A value less than or equals to zero
# leads to disabling this caching. This caching helps more if transactions
# perform significant writes.
bucketCacheSize: 100

# configurations for 'trie'
# 'tire' has no additional configurations exposed as yet
# stateDatabase - options are "goleveldb", "CouchDB"
# goleveldb - default state database stored in goleveldb.
# CouchDB - store state database in CouchDB
stateDatabase: goleveldb
couchDBConfig:
couchDBAddress: 127.0.0.1:5984
username:
password:

# historyDatabase - options are true or false
# Indicates if the history of key updates should be stored in goleveldb
historyDatabase: true

# Limit on the number of records to return per query
queryLimit: 10000


################################################################################
Expand Down
236 changes: 228 additions & 8 deletions core/chaincode/exectransaction_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package chaincode

import (
"encoding/json"
"fmt"
"net"
"os"
Expand All @@ -35,6 +36,7 @@ import (
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
"github.com/hyperledger/fabric/core/ledger/ledgermgmt"
"github.com/hyperledger/fabric/core/ledger/util/couchdb"
"github.com/hyperledger/fabric/core/peer"
"github.com/hyperledger/fabric/core/scc"
"github.com/hyperledger/fabric/msp"
Expand Down Expand Up @@ -112,6 +114,22 @@ func finitPeer(lis net.Listener, chainIDs ...string) {
ledgerPath := viper.GetString("peer.fileSystemPath")
os.RemoveAll(ledgerPath)
os.RemoveAll(filepath.Join(os.TempDir(), "hyperledger"))

//if couchdb is enabled, then cleanup the test couchdb
if ledgerconfig.IsCouchDBEnabled() == true {

chainID := util.GetTestChainID()

connectURL := viper.GetString("ledger.state.couchDBConfig.couchDBAddress")
username := viper.GetString("ledger.state.couchDBConfig.username")
password := viper.GetString("ledger.state.couchDBConfig.password")

couchInstance, _ := couchdb.CreateCouchInstance(connectURL, username, password)
db, _ := couchdb.CreateCouchDatabase(*couchInstance, chainID)
//drop the test database
db.DropDatabase()

}
}

func startTxSimulation(ctxt context.Context, chainID string) (context.Context, ledger.TxSimulator, error) {
Expand Down Expand Up @@ -911,9 +929,10 @@ func TestQueries(t *testing.T) {
return
}

// Invoke second chaincode, which will inturn invoke the first chaincode
// Add 12 marbles for testing range queries and rich queries (for capable ledgers)
// The tests will test both range and rich queries and queries with query limits
f = "put"
args = util.ToChaincodeArgs(f, "key1", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
args = util.ToChaincodeArgs(f, "marble01", "{\"docType\":\"marble\",\"name\":\"marble01\",\"color\":\"blue\",\"size\":35,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
Expand All @@ -926,7 +945,7 @@ func TestQueries(t *testing.T) {
}

f = "put"
args = util.ToChaincodeArgs(f, "key2", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
args = util.ToChaincodeArgs(f, "marble02", "{\"docType\":\"marble\",\"name\":\"marble02\",\"color\":\"red\",\"size\":25,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
Expand All @@ -939,7 +958,7 @@ func TestQueries(t *testing.T) {
}

f = "put"
args = util.ToChaincodeArgs(f, "key3", "{\"shipmentID\":\"161003PKC7300\",\"customsInvoice\":{\"methodOfTransport\":\"GROUND\",\"invoiceNumber\":\"00091622\"},\"weightUnitOfMeasure\":\"KGM\",\"volumeUnitOfMeasure\": \"CO\",\"dimensionUnitOfMeasure\":\"CM\",\"currency\":\"USD\"}")
args = util.ToChaincodeArgs(f, "marble03", "{\"docType\":\"marble\",\"name\":\"marble03\",\"color\":\"green\",\"size\":15,\"owner\":\"tom\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
Expand All @@ -950,9 +969,92 @@ func TestQueries(t *testing.T) {
return
}

f = "keys"
args = util.ToChaincodeArgs(f, "key0", "key3")
f = "put"
args = util.ToChaincodeArgs(f, "marble04", "{\"docType\":\"marble\",\"name\":\"marble04\",\"color\":\"green\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble05", "{\"docType\":\"marble\",\"name\":\"marble05\",\"color\":\"red\",\"size\":25,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble06", "{\"docType\":\"marble\",\"name\":\"marble06\",\"color\":\"blue\",\"size\":35,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble07", "{\"docType\":\"marble\",\"name\":\"marble07\",\"color\":\"yellow\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble08", "{\"docType\":\"marble\",\"name\":\"marble08\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble09", "{\"docType\":\"marble\",\"name\":\"marble09\",\"color\":\"yellow\",\"size\":10,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble10", "{\"docType\":\"marble\",\"name\":\"marble10\",\"color\":\"red\",\"size\":20,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble11", "{\"docType\":\"marble\",\"name\":\"marble11\",\"color\":\"green\",\"size\":40,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
Expand All @@ -963,19 +1065,137 @@ func TestQueries(t *testing.T) {
return
}

f = "put"
args = util.ToChaincodeArgs(f, "marble12", "{\"docType\":\"marble\",\"name\":\"marble12\",\"color\":\"red\",\"size\":30,\"owner\":\"jerry\"}")
spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//TODO - the following query tests for queryLimits may change due to future designs
// for batch "paging"

//The following range query for "marble01" to "marble11" should return 10 marbles
f = "keys"
args = util.ToChaincodeArgs(f, "marble01", "marble11")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err := invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

var keys []interface{}
err = json.Unmarshal(retval, &keys)

//default query limit of 10000 is used, query should return all records that meet the criteria
if len(keys) != 10 {
t.Fail()
t.Logf("Error detected with the range query, should have returned 10 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//Reset the query limit to 5
viper.Set("ledger.state.queryLimit", 5)

//The following range query for "marble01" to "marble11" should return 5 marbles due to the queryLimit
f = "keys"
args = util.ToChaincodeArgs(f, "marble01", "marble11")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)

nextBlockNumber++
if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 5 values
if len(keys) != 5 {
t.Fail()
t.Logf("Error detected with the range query, should have returned 5 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//Reset the query limit to default
viper.Set("ledger.state.queryLimit", 10000)

if ledgerconfig.IsCouchDBEnabled() == true {

//The following rich query for should return 9 marbles
f = "query"
args = util.ToChaincodeArgs(f, "{\"selector\":{\"owner\":\"jerry\"}}")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)
nextBlockNumber++

if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 9 values
//default query limit of 10000 is used, this query is effectively unlimited
if len(keys) != 9 {
t.Fail()
t.Logf("Error detected with the rich query, should have returned 9 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//Reset the query limit to 5
viper.Set("ledger.state.queryLimit", 5)

//The following rich query should return 5 marbles due to the queryLimit
f = "query"
args = util.ToChaincodeArgs(f, "{\"selector\":{\"currency\":\"USD\"}}")
args = util.ToChaincodeArgs(f, "{\"selector\":{\"owner\":\"jerry\"}}")

spec = &pb.ChaincodeSpec{Type: 1, ChaincodeId: cID, Input: &pb.ChaincodeInput{Args: args}}
_, _, _, err = invoke(ctxt, chainID, spec, nextBlockNumber)
_, _, retval, err = invoke(ctxt, chainID, spec, nextBlockNumber)

if err != nil {
t.Fail()
t.Logf("Error invoking <%s>: %s", ccID, err)
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

//unmarshal the results
err = json.Unmarshal(retval, &keys)

//check to see if there are 5 values
if len(keys) != 5 {
t.Fail()
t.Logf("Error detected with the rich query, should have returned 5 but returned %v", len(keys))
theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
return
}

}

theChaincodeSupport.Stop(ctxt, cccid, &pb.ChaincodeDeploymentSpec{ChaincodeSpec: spec})
}

Expand Down
Loading

0 comments on commit 0fc6c4d

Please sign in to comment.