Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CouchDB index support for implicit collections #4794

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

CouchDB index support for implicit collections #4794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4dc59a7
enabled support for couchDB indexes on implicit collections
bkiran6398 Apr 3, 2024
7b0a4bf
updated all org implicit collection notation to _implicit_all_orgs
bkiran6398 Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion core/chaincode/implicitcollection/name.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import (
)

const (
prefix = "_implicit_org_"
prefix = "_implicit_org_"
allOrgNotation = "_implicit_all_orgs"
)

// NameForOrg constructs the name of the implicit collection for the specified org
Expand All @@ -31,3 +32,7 @@ func MspIDIfImplicitCollection(collectionName string) (isImplicitCollection bool
func IsImplicitCollection(collectionName string) bool {
return strings.HasPrefix(collectionName, prefix)
}

func IsAllOrgNotation(collectionName string) bool {
return collectionName == allOrgNotation
}
1 change: 1 addition & 0 deletions core/ledger/kvledger/kv_ledger_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ func (p *Provider) initStateDBProvider() error {
p.initializer.HealthCheckRegistry,
stateDBConfig,
sysNamespaces,
p.initializer.MembershipInfoProvider.MyImplicitCollectionName(),
)
return err
}
Expand Down
33 changes: 20 additions & 13 deletions core/ledger/kvledger/txmgmt/privacyenabledstate/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/hyperledger/fabric-lib-go/common/flogging"
"github.com/hyperledger/fabric-lib-go/common/metrics"
"github.com/hyperledger/fabric-lib-go/healthz"
"github.com/hyperledger/fabric/core/chaincode/implicitcollection"
"github.com/hyperledger/fabric/core/common/ccprovider"
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/cceventmgmt"
Expand Down Expand Up @@ -46,9 +47,10 @@ type StateDBConfig struct {
// DBProvider encapsulates other providers such as VersionedDBProvider and
// BookeepingProvider which are required to create DB for a channel
type DBProvider struct {
VersionedDBProvider statedb.VersionedDBProvider
HealthCheckRegistry ledger.HealthCheckRegistry
bookkeepingProvider *bookkeeping.Provider
VersionedDBProvider statedb.VersionedDBProvider
HealthCheckRegistry ledger.HealthCheckRegistry
bookkeepingProvider *bookkeeping.Provider
myImplicitCollection string
}

// NewDBProvider constructs an instance of DBProvider
Expand All @@ -58,6 +60,7 @@ func NewDBProvider(
healthCheckRegistry ledger.HealthCheckRegistry,
stateDBConf *StateDBConfig,
sysNamespaces []string,
myImplicitCollection string,
) (*DBProvider, error) {
var vdbProvider statedb.VersionedDBProvider
var err error
Expand All @@ -73,9 +76,10 @@ func NewDBProvider(
}

dbProvider := &DBProvider{
VersionedDBProvider: vdbProvider,
HealthCheckRegistry: healthCheckRegistry,
bookkeepingProvider: bookkeeperProvider,
VersionedDBProvider: vdbProvider,
HealthCheckRegistry: healthCheckRegistry,
bookkeepingProvider: bookkeeperProvider,
myImplicitCollection: myImplicitCollection,
}

err = dbProvider.RegisterHealthChecker()
Expand Down Expand Up @@ -107,7 +111,7 @@ func (p *DBProvider) GetDBHandle(id string, chInfoProvider channelInfoProvider)
if err != nil {
return nil, err
}
return NewDB(vdb, id, metadataHint)
return NewDB(vdb, id, metadataHint, p.myImplicitCollection)
}

// Close closes all the VersionedDB instances and releases any resources held by VersionedDBProvider
Expand All @@ -123,13 +127,14 @@ func (p *DBProvider) Drop(ledgerid string) error {
// DB uses a single database to maintain both the public and private data
type DB struct {
statedb.VersionedDB
metadataHint *metadataHint
metadataHint *metadataHint
myImplicitCollection string
}

// NewDB wraps a VersionedDB instance. The public data is managed directly by the wrapped versionedDB.
// For managing the hashed data and private data, this implementation creates separate namespaces in the wrapped db
func NewDB(vdb statedb.VersionedDB, ledgerid string, metadataHint *metadataHint) (*DB, error) {
return &DB{vdb, metadataHint}, nil
func NewDB(vdb statedb.VersionedDB, ledgerid string, metadataHint *metadataHint, myImplicitCollection string) (*DB, error) {
return &DB{vdb, metadataHint, myImplicitCollection}, nil
}

// IsBulkOptimizable checks whether the underlying statedb implements statedb.BulkOptimizable
Expand Down Expand Up @@ -330,9 +335,11 @@ func (s *DB) HandleChaincodeDeploy(chaincodeDefinition *cceventmgmt.ChaincodeDef
case indexInfo.hasIndexForCollection:
_, ok := collectionConfigMap[indexInfo.collectionName]
if !ok {
logger.Errorf("Error processing index for chaincode [%s]: cannot create an index for an undefined collection=[%s]",
chaincodeDefinition.Name, indexInfo.collectionName)
continue
if !implicitcollection.IsAllOrgNotation(indexInfo.collectionName) && indexInfo.collectionName != s.myImplicitCollection {
logger.Debugf("Skipped processing index of other org implicit collection=[%s] for chaincode [%s] ",
indexInfo.collectionName, chaincodeDefinition.Name)
continue
}
}
err := indexCapable.ProcessIndexesForChaincodeDeploy(derivePvtDataNs(chaincodeDefinition.Name, indexInfo.collectionName), indexFilesData)
if err != nil {
Expand Down
25 changes: 23 additions & 2 deletions core/ledger/kvledger/txmgmt/privacyenabledstate/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"encoding/hex"
"fmt"
"io"
"os"
"testing"

"github.com/hyperledger/fabric-protos-go/peer"
Expand Down Expand Up @@ -438,6 +439,9 @@ func createCollectionConfig(collectionName string) *peer.CollectionConfig {
func testHandleChainCodeDeploy(t *testing.T, env TestEnv) {
env.Init(t)
defer env.Cleanup()
// Set local MSP ID for DB to use
err := os.Setenv("CORE_PEER_LOCALMSPID", "testOrgMsp1")
require.NoError(t, err)
db := env.GetDBHandle(generateLedgerID(t))

coll1 := createCollectionConfig("collectionMarbles")
Expand All @@ -451,29 +455,46 @@ func testHandleChainCodeDeploy(t *testing.T, env TestEnv) {
{Name: "META-INF/statedb/couchdb/indexes/indexSizeSortName.json", Body: `{"index":{"fields":[{"size":"desc"}]},"ddoc":"indexSizeSortName","name":"indexSizeSortName","type":"json"}`},
{Name: "META-INF/statedb/couchdb/collections/collectionMarbles/indexes/indexCollMarbles.json", Body: `{"index":{"fields":["docType","owner"]},"ddoc":"indexCollectionMarbles", "name":"indexCollectionMarbles","type":"json"}`},
{Name: "META-INF/statedb/couchdb/collections/collectionMarblesPrivateDetails/indexes/indexCollPrivDetails.json", Body: `{"index":{"fields":["docType","price"]},"ddoc":"indexPrivateDetails", "name":"indexPrivateDetails","type":"json"}`},
{Name: "META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp1/indexes/indexCreateDate.json", Body: `{"index":{"fields":["create_date"]},"ddoc":"indexCreateDateDoc", "name":"indexCreateDate","type":"json"}`},
{Name: "META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp2/indexes/indexUpdateDate.json", Body: `{"index":{"fields":["update_date"]},"ddoc":"indexUpdateDateDoc", "name":"indexUpdateDate","type":"json"}`},
{Name: "META-INF/statedb/couchdb/collections/_implicit_org_*/indexes/indexDeleteDate.json", Body: `{"index":{"fields":["delete_date"]},"ddoc":"indexDeleteDateDoc", "name":"indexDeleteDate","type":"json"}`},
},
)

// Test the retrieveIndexArtifacts method
fileEntries, err := ccprovider.ExtractFileEntries(dbArtifactsTarBytes, "couchdb")
require.NoError(t, err)

// There should be 3 entries
require.Len(t, fileEntries, 3)
// There should be 6 entries
require.Len(t, fileEntries, 6)

// There should be 2 entries for main
require.Len(t, fileEntries["META-INF/statedb/couchdb/indexes"], 2)

// There should be 1 entry for collectionMarbles
require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/collectionMarbles/indexes"], 1)

// There should be 1 entry for _implicit_org_testOrgMsp1
require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp1/indexes"], 1)

// There should be 1 entry for _implicit_org_testOrgMsp2
require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp2/indexes"], 1)

// There should be 1 entry for _implicit_org_*
require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_*/indexes"], 1)

// Verify the content of the array item
expectedJSON := []byte(`{"index":{"fields":["docType","owner"]},"ddoc":"indexCollectionMarbles", "name":"indexCollectionMarbles","type":"json"}`)
actualJSON := fileEntries["META-INF/statedb/couchdb/collections/collectionMarbles/indexes"][0].FileContent
require.Equal(t, expectedJSON, actualJSON)

// The collection config is added to the chaincodeDef but missing collectionMarblesPrivateDetails.
// Hence, the index on collectionMarblesPrivateDetails cannot be created
//
// Index on _implicit_org_testOrgMsp1 will be created as it is implicit collection of this org,
// but _implicit_org_testOrgMsp2 will not be created as it belongs to a different org
//
// Index on _implicit_org_* will be created as it applies to all orgs' implicit collections
err = db.HandleChaincodeDeploy(chaincodeDef, dbArtifactsTarBytes)
require.NoError(t, err)

Expand Down
11 changes: 5 additions & 6 deletions docs/source/private-data-arch.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,11 @@ The properties ``requiredPeerCount`` and ``maxPeerCount`` can however be set in
can set these properties based on the number of peers that they deploy, as described
in the next section.

.. note:: Since implicit private data collections are not explicitly defined,
it is not possible to associate CouchDB indexes with them. Utilize
key-based queries and key-range queries rather than JSON queries.
.. note::

CouchDB indexes can be created to organisation specific implicit collections just like explicitly defined collections with indexes directory name `_implicit_org_<MSP_ID>`. Such indexes are only effective within the respective organization to which the implicit collection belongs.

To create common indexes for implicit collection of all the organisations, the indexes directory should be named "_implicit_org_*".

Private data dissemination
--------------------------
Expand Down Expand Up @@ -361,9 +363,6 @@ Limitations:
chaincode function to make the updates. Note that calls to GetPrivateData() to retrieve
individual keys can be made in the same transaction as PutPrivateData() calls, since
all peers can validate key reads based on the hashed key version.
* Since implicit private data collections are not explicitly defined,
it is not possible to associate CouchDB indexes with them.
It is therefore not recommended to utilize JSON queries with implicit private data collections.

Using Indexes with collections
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand Down
Loading