TransactionManager can be initialized but unusable if other components are not initialized

[pkoenig10]

See internal link: https://g.p.b/foundry/mp/pull/12089#discussion_r1462867

AtlasDB has a number of components which need to be initialized (one of which is the transaction manager). If the initialization does not succeed on the first attempt, then another attempt will be scheduled for 10 second later.

atlasdb/atlasdb-commons/src/main/java/com/palantir/async/initializer/AsyncInitializer.java

Lines 89 to 102 in b0e4331

	// Not final for tests.
	void scheduleInitialization() {
	singleThreadedExecutor.schedule(
	() -> {
	if (state.isCancelled()) {
	singleThreadedExecutor.shutdown();
	return;
	}
	tryInitializationLoop();
	},
	sleepIntervalInMillis(),
	TimeUnit.MILLISECONDS);
	}

Unfortunately, the initializations are all independent. This means that the transaction manager can be initialized but unusable if another required component (like the PersistentTimestampServiceImpl in the example above) is not yet initialized.

com.palantir.exception.NotInitializedException: The PersistentTimestampService is not initialized yet
	at com.palantir.async.initializer.AsyncInitializer.checkInitialized(AsyncInitializer.java:138)
	at com.palantir.timestamp.PersistentTimestampServiceImpl$InitializingWrapper.delegate(PersistentTimestampServiceImpl.java:33)
	at com.palantir.timestamp.AutoDelegate_PersistentTimestampService.getFreshTimestamp(AutoDelegate_PersistentTimestampService.java:36)
	at jdk.internal.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.leader.proxy.AwaitingLeadershipProxy.handleInvocation(AwaitingLeadershipProxy.java:134)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy89.getFreshTimestamp(Unknown Source)
	at jdk.internal.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.atlasdb.factory.LocalOrRemoteProxy.handleInvocation(LocalOrRemoteProxy.java:64)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy91.getFreshTimestamp(Unknown Source)
	at com.palantir.lock.impl.LegacyTimelockService.lockImmutableTimestamp(LegacyTimelockService.java:87)
	at com.palantir.lock.impl.LegacyTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$0(LegacyTimelockService.java:123)
	at java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:104)
	at java.util.stream.IntPipeline$Head.forEach(IntPipeline.java:593)
	at com.palantir.lock.impl.LegacyTimelockService.startIdentifiedAtlasDbTransactionBatch(LegacyTimelockService.java:122)
	at com.palantir.lock.client.TimeLockClient.lambda$startIdentifiedAtlasDbTransactionBatch$0(TimeLockClient.java:112)
	at com.palantir.lock.client.TimeLockClient.executeOnTimeLock(TimeLockClient.java:173)
	at com.palantir.lock.client.TimeLockClient.startIdentifiedAtlasDbTransactionBatch(TimeLockClient.java:112)
	at com.palantir.lock.client.ProfilingTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$2(ProfilingTimelockService.java:122)
	at com.palantir.lock.client.ProfilingTimelockService.runTaskTimed(ProfilingTimelockService.java:174)
	at com.palantir.lock.client.ProfilingTimelockService.startIdentifiedAtlasDbTransactionBatch(ProfilingTimelockService.java:121)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$3(TimestampCorroboratingTimelockService.java:76)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.checkAndUpdateLowerBound(TimestampCorroboratingTimelockService.java:94)
	at com.palantir.atlasdb.factory.timelock.TimestampCorroboratingTimelockService.startIdentifiedAtlasDbTransactionBatch(TimestampCorroboratingTimelockService.java:75)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.lambda$startIdentifiedAtlasDbTransactionBatch$2(InstrumentedTimelockService.java:88)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.executeWithRecord(InstrumentedTimelockService.java:136)
	at com.palantir.atlasdb.transaction.impl.InstrumentedTimelockService.startIdentifiedAtlasDbTransactionBatch(InstrumentedTimelockService.java:88)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.startTransactions(SnapshotTransactionManager.java:182)
	at com.palantir.atlasdb.transaction.impl.SerializableTransactionManager.startTransactions(SerializableTransactionManager.java:62)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.lambda$runTaskWithConditionThrowOnConflict$1(SnapshotTransactionManager.java:168)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.runTimed(SnapshotTransactionManager.java:517)
	at com.palantir.atlasdb.transaction.impl.SnapshotTransactionManager.runTaskWithConditionThrowOnConflict(SnapshotTransactionManager.java:167)
	at com.palantir.atlasdb.transaction.impl.SerializableTransactionManager.runTaskWithConditionThrowOnConflict(SerializableTransactionManager.java:62)
	at com.palantir.atlasdb.transaction.impl.AbstractConditionAwareTransactionManager.lambda$runTaskWithConditionWithRetry$0(AbstractConditionAwareTransactionManager.java:50)
	at com.github.rholder.retry.AttemptTimeLimiters$NoAttemptTimeLimit.call(AttemptTimeLimiters.java:78)
	at com.github.rholder.retry.Retryer.call(Retryer.java:160)
	at com.palantir.atlasdb.transaction.impl.TransactionRetryStrategy.runWithRetry(TransactionRetryStrategy.java:84)
	at com.palantir.atlasdb.transaction.impl.AbstractConditionAwareTransactionManager.runTaskWithConditionWithRetry(AbstractConditionAwareTransactionManager.java:47)
	at com.palantir.tritium.proxy.InstrumentedTransactionManager$8.runTaskWithConditionWithRetry(Unknown Source)
	at com.palantir.atlas.migration2.api.MigrationAwareTransactionManagerImpl.runTaskWithConditionWithRetry(MigrationAwareTransactionManagerImpl.java:126)
	at com.palantir.atlas.migration2.api.MigrationAwareTransactionManagerImpl.runTaskWithRetry(MigrationAwareTransactionManagerImpl.java:131)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:564)
	at com.palantir.atlas.migration2.api.QosAwareProxy.handleInvocation(QosAwareProxy.java:34)
	at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86)
	at com.sun.proxy.$Proxy186.runTaskWithRetry(Unknown Source)