Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-11555. SCMDBDefinition should be singleton. #7296

Merged
merged 1 commit into from
Oct 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@
package org.apache.hadoop.hdds.scm.ha;

import java.io.IOException;

import com.google.common.base.Preconditions;
import java.util.Objects;

import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointRequestProto;
import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointResponseProto;
Expand Down Expand Up @@ -52,12 +51,11 @@ public class InterSCMGrpcService extends
private final Table<String, TransactionInfo> transactionInfoTable;

InterSCMGrpcService(final StorageContainerManager scm) throws IOException {
Preconditions.checkNotNull(scm);
Objects.requireNonNull(scm, "scm");
this.scm = scm;
this.transactionInfoTable = HAUtils.getTransactionInfoTable(
scm.getScmMetadataStore().getStore(), new SCMDBDefinition());
provider =
new SCMDBCheckpointProvider(scm.getScmMetadataStore().getStore());
scm.getScmMetadataStore().getStore(), SCMDBDefinition.get());
this.provider = new SCMDBCheckpointProvider(scm.getScmMetadataStore().getStore());
}

@Override
Expand All @@ -67,7 +65,7 @@ public void download(CopyDBCheckpointRequestProto request,
scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
TransactionInfo transactionInfo =
transactionInfoTable.get(TRANSACTION_INFO_KEY);
Preconditions.checkNotNull(transactionInfo);
Objects.requireNonNull(transactionInfo, "transactionInfo");
SCMGrpcOutputStream outputStream =
new SCMGrpcOutputStream(responseObserver, scm.getClusterId(),
BUFFER_SIZE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public class SCMHAManagerImpl implements SCMHAManager {

private final SCMRatisServer ratisServer;
private final ConfigurationSource conf;
private final OzoneConfiguration ozoneConf;
private final SecurityConfig securityConfig;
private final DBTransactionBuffer transactionBuffer;
private final SCMSnapshotProvider scmSnapshotProvider;
Expand All @@ -89,6 +90,7 @@ public SCMHAManagerImpl(final ConfigurationSource conf,
final SecurityConfig securityConfig,
final StorageContainerManager scm) throws IOException {
this.conf = conf;
this.ozoneConf = OzoneConfiguration.of(conf);
this.securityConfig = securityConfig;
this.scm = scm;
this.exitManager = new ExitManager();
Expand Down Expand Up @@ -128,7 +130,7 @@ public void start() throws IOException {
// It will first try to add itself to existing ring
final SCMNodeDetails nodeDetails =
scm.getSCMHANodeDetails().getLocalNodeDetails();
final boolean success = HAUtils.addSCM(OzoneConfiguration.of(conf),
final boolean success = HAUtils.addSCM(ozoneConf,
new AddSCMRequest.Builder().setClusterId(scm.getClusterId())
.setScmId(scm.getScmId())
.setRatisAddr(nodeDetails
Expand Down Expand Up @@ -221,17 +223,18 @@ public List<ManagedSecretKey> getSecretKeysFromLeader(String leaderID)
}
}

private TransactionInfo getTransactionInfoFromCheckpoint(Path checkpointLocation) throws IOException {
return HAUtils.getTrxnInfoFromCheckpoint(
ozoneConf, checkpointLocation, SCMDBDefinition.get());
}

@Override
public TermIndex verifyCheckpointFromLeader(String leaderId,
DBCheckpoint checkpoint) {
try {
Path checkpointLocation = checkpoint.getCheckpointLocation();
TransactionInfo checkpointTxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(OzoneConfiguration.of(conf),
checkpointLocation, new SCMDBDefinition());

LOG.info("Installing checkpoint with SCMTransactionInfo {}",
checkpointTxnInfo);
final TransactionInfo checkpointTxnInfo = getTransactionInfoFromCheckpoint(checkpointLocation);
LOG.info("{}: Verify checkpoint {} from leader {}", scm.getScmId(), checkpointTxnInfo, leaderId);

TermIndex termIndex =
getRatisServer().getSCMStateMachine().getLastAppliedTermIndex();
Expand Down Expand Up @@ -281,12 +284,9 @@ public TermIndex installCheckpoint(DBCheckpoint dbCheckpoint)
throws Exception {

Path checkpointLocation = dbCheckpoint.getCheckpointLocation();
TransactionInfo checkpointTrxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(OzoneConfiguration.of(conf),
checkpointLocation, new SCMDBDefinition());
final TransactionInfo checkpointTrxnInfo = getTransactionInfoFromCheckpoint(checkpointLocation);

LOG.info("Installing checkpoint with SCMTransactionInfo {}",
checkpointTrxnInfo);
LOG.info("{}: Install checkpoint {}", scm.getScmId(), checkpointTrxnInfo);

return installCheckpoint(checkpointLocation, checkpointTrxnInfo);
}
Expand Down Expand Up @@ -457,7 +457,7 @@ public void startServices() throws IOException {

// TODO: Fix the metrics ??
final SCMMetadataStore metadataStore = scm.getScmMetadataStore();
metadataStore.start(OzoneConfiguration.of(conf));
metadataStore.start(ozoneConf);
scm.getSequenceIdGen().reinitialize(metadataStore.getSequenceIdTable());
scm.getPipelineManager().reinitialize(metadataStore.getPipelineTable());
scm.getContainerManager().reinitialize(metadataStore.getContainerTable());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,6 @@
* Class defines the structure and types of the scm.db.
*/
public class SCMDBDefinition extends DBDefinition.WithMap {
public SCMDBDefinition() {
this(COLUMN_FAMILIES);
}

protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
super(map);
}

public static final DBColumnFamilyDefinition<Long, DeletedBlocksTransaction>
DELETED_BLOCKS =
new DBColumnFamilyDefinition<>(
Expand Down Expand Up @@ -156,6 +148,16 @@ protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
VALID_CERTS,
VALID_SCM_CERTS);

private static final SCMDBDefinition INSTANCE = new SCMDBDefinition(COLUMN_FAMILIES);

public static SCMDBDefinition get() {
return INSTANCE;
}

protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
super(map);
}

@Override
public String getName() {
return "scm.db";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public SCMMetadataStoreImpl(OzoneConfiguration config)
public void start(OzoneConfiguration config)
throws IOException {
if (this.store == null) {
SCMDBDefinition scmdbDefinition = new SCMDBDefinition();
final SCMDBDefinition scmdbDefinition = SCMDBDefinition.get();
File metaDir = HAUtils.getMetaDir(scmdbDefinition, configuration);
// Check if there is a DB Inconsistent Marker in the metaDir. This
// marker indicates that the DB is in an inconsistent state and hence
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ public class TestContainerManagerImpl {
@BeforeEach
void setUp() throws Exception {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
nodeManager = new MockNodeManager(true, 10);
sequenceIdGen = new SequenceIdGenerator(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,7 @@ void setup() throws IOException, InvalidStateTransitionException {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
nodeManager = new MockNodeManager(true, 10);
containerManager = mock(ContainerManager.class);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,7 @@ public void init() throws IOException, TimeoutException {
OzoneConfiguration conf = new OzoneConfiguration();
scmhaManager = SCMHAManagerStub.getInstance(true);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
pipelineManager = mock(PipelineManager.class);
pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.CLOSED)
.setId(PipelineID.randomId())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,7 @@ public void setup() throws IOException, InvalidStateTransitionException,
new SCMNodeManager(conf, storageConfig, eventQueue, clusterMap,
scmContext, versionManager);
scmhaManager = SCMHAManagerStub.getInstance(true);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());

pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ public void setup() throws IOException {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
this.nodeManager = new MockNodeManager(true, 10);
this.containerManager = mock(ContainerManager.class);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,7 @@ void setup(@TempDir File testDir) throws IOException, InterruptedException,
nodeManager = new SimpleMockNodeManager();
eventQueue = new EventQueue();
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
PipelineManager pipelineManager = mock(PipelineManager.class);
when(pipelineManager.containsPipeline(any(PipelineID.class)))
.thenReturn(true);
Expand Down Expand Up @@ -277,8 +276,7 @@ private void createReplicationManager(ReplicationManagerConfiguration rmConf,

SCMHAManager scmHAManager = SCMHAManagerStub
.getInstance(true, new SCMDBTransactionBufferImpl());
dbStore = DBStoreBuilder.createDBStore(
config, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(config, SCMDBDefinition.get());

LegacyReplicationManager legacyRM = new LegacyReplicationManager(
config, containerManager, ratisContainerPlacementPolicy, eventQueue,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class TestStatefulServiceStateManagerImpl {
void setup(@TempDir File testDir) throws IOException {
conf = SCMTestUtils.getConf(testDir);
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
statefulServiceConfig =
SCMDBDefinition.STATEFUL_SERVICE_CONFIG.getTable(dbStore);
scmhaManager = SCMHAManagerStub.getInstance(true, dbStore);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@ public class TestContainerPlacement {
public void setUp() throws Exception {
conf = getConf();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
sequenceIdGen = new SequenceIdGenerator(
conf, scmhaManager, SCMDBDefinition.SEQUENCE_ID.getTable(dbStore));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ public class TestPipelineDatanodesIntersection {
public void initialize() throws IOException {
conf = SCMTestUtils.getConf(testDir);
end = false;
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
}

@AfterEach
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ void init(@TempDir File testDir, @TempDir File dbDir) throws Exception {
// placement policy (Rack Scatter), so just use the random one.
conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY,
SCMContainerPlacementRandom.class.getName());
dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
nodeManager = new MockNodeManager(true, 20);
maxPipelineCount = nodeManager.getNodeCount(
HddsProtos.NodeOperationalState.IN_SERVICE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,7 @@ private void setupRacks(int datanodeCount, int nodesPerRack,
.thenReturn(dn);
}

dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);

stateManager = PipelineStateManagerImpl.newBuilder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ public void init() throws Exception {
conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
10, StorageUnit.MB);
nodeManager.setNumPipelinePerDatanode(PIPELINE_LOAD_LIMIT);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
stateManager = PipelineStateManagerImpl.newBuilder()
.setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ public class TestPipelineStateManagerImpl {
@BeforeEach
public void init() throws Exception {
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());

SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
NodeManager nodeManager = new MockNodeManager(true, 10);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,7 @@ public void init(int maxPipelinePerNode, OzoneConfiguration conf)

public void init(int maxPipelinePerNode, OzoneConfiguration conf, File dir) throws Exception {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
nodeManager = new MockNodeManager(true, 10);
nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode);
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,7 @@ public class TestSimplePipelineProvider {
public void init() throws Exception {
nodeManager = new MockNodeManager(true, 10);
final OzoneConfiguration conf = SCMTestUtils.getConf(testDir);
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true);
stateManager = PipelineStateManagerImpl.newBuilder()
.setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ void setup(@TempDir File testDir) throws IOException {

containers = new HashMap<>();
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
dbStore = DBStoreBuilder.createDBStore(
conf, new SCMDBDefinition());
dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get());
scmhaManager = SCMHAManagerStub.getInstance(true);
pipelineManager =
new MockPipelineManager(dbStore, scmhaManager, nodeManager);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,14 @@ private DBCheckpoint downloadSnapshot() throws Exception {
public void testInstallCheckPoint() throws Exception {
DBCheckpoint checkpoint = downloadSnapshot();
StorageContainerManager scm = cluster.getStorageContainerManager();
DBStore db = HAUtils
.loadDB(conf, checkpoint.getCheckpointLocation().getParent().toFile(),
checkpoint.getCheckpointLocation().getFileName().toString(),
new SCMDBDefinition());
final Path location = checkpoint.getCheckpointLocation();
final DBStore db = HAUtils.loadDB(conf, location.getParent().toFile(),
location.getFileName().toString(), SCMDBDefinition.get());
// Hack the transaction index in the checkpoint so as to ensure the
// checkpointed transaction index is higher than when it was downloaded
// from.
assertNotNull(db);
HAUtils.getTransactionInfoTable(db, new SCMDBDefinition())
HAUtils.getTransactionInfoTable(db, SCMDBDefinition.get())
.put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(10, 100));
db.close();
ContainerID cid =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,8 @@ public void testInstallCorruptedCheckpointFailure() throws Exception {
DBCheckpoint leaderDbCheckpoint = leaderSCM.getScmMetadataStore().getStore()
.getCheckpoint(false);
Path leaderCheckpointLocation = leaderDbCheckpoint.getCheckpointLocation();
TransactionInfo leaderCheckpointTrxnInfo = HAUtils
.getTrxnInfoFromCheckpoint(conf, leaderCheckpointLocation,
new SCMDBDefinition());
final TransactionInfo leaderCheckpointTrxnInfo = HAUtils.getTrxnInfoFromCheckpoint(
conf, leaderCheckpointLocation, SCMDBDefinition.get());

assertNotNull(leaderCheckpointLocation);
// Take a backup of the current DB
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class ReconSCMDBDefinition extends SCMDBDefinition {

private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
new SCMDBDefinition().getMap(), NODES);
SCMDBDefinition.get().getMap(), NODES);

public ReconSCMDBDefinition() {
super(COLUMN_FAMILIES);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ private DBDefinitionFactory() {

static {
final Map<String, DBDefinition> map = new HashMap<>();
Arrays.asList(new SCMDBDefinition(), OMDBDefinition.get(), new ReconSCMDBDefinition())
Arrays.asList(SCMDBDefinition.get(), OMDBDefinition.get(), new ReconSCMDBDefinition())
.forEach(dbDefinition -> map.put(dbDefinition.getName(), dbDefinition));
DB_MAP = Collections.unmodifiableMap(map);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public Void call() throws Exception {

ConfigurationSource config = createOzoneConfiguration();

scmDb = DBStoreBuilder.createDBStore(config, new SCMDBDefinition());
scmDb = DBStoreBuilder.createDBStore(config, SCMDBDefinition.get());

containerStore = CONTAINERS.getTable(scmDb);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@ public void testGetDefinition() {
DBDefinition definition = DBDefinitionFactory.getDefinition(OMDBDefinition.get().getName());
assertInstanceOf(OMDBDefinition.class, definition);

definition = DBDefinitionFactory.getDefinition(
new SCMDBDefinition().getName());
definition = DBDefinitionFactory.getDefinition(SCMDBDefinition.get().getName());
assertInstanceOf(SCMDBDefinition.class, definition);

definition = DBDefinitionFactory.getDefinition(
Expand Down