From 70e73bc386d419e3f6a5a37e52a6ed5a0d52b4ea Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Wed, 27 Nov 2024 18:07:22 -0500 Subject: [PATCH 1/2] Fix flaky FateIT.testTransactionStatus() (#5121) Related to #2474 and #2550. The sync fix in #2550 helped ensure that the ZooKeeper client had the updated status, it is still possible for the node to be deleted before the SUCCESSFUL transaction status is observed. This change makes FateIT more stable by accepting that as a valid possible outcome, and no longer fails when that happens. --- .../accumulo/test/fate/zookeeper/FateIT.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java index 2dde8fabcaf..bc92312d415 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java @@ -19,12 +19,12 @@ package org.apache.accumulo.test.fate.zookeeper; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.FAILED; import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS; import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.IN_PROGRESS; import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.NEW; import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.SUBMITTED; -import static org.apache.accumulo.core.fate.ReadOnlyTStore.TStatus.SUCCESSFUL; import static org.apache.accumulo.harness.AccumuloITBase.ZOOKEEPER_TESTING_SERVER; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; @@ -40,6 +40,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.clientImpl.thrift.TableOperation; @@ -243,27 +244,39 @@ public void testTransactionStatus() throws Exception { assertEquals(IN_PROGRESS, getTxStatus(zk, txid)); // tell the op to exit the method finishCall.countDown(); - // Check that it transitions to SUCCESSFUL - TStatus s = getTxStatus(zk, txid); - while (s != SUCCESSFUL) { - s = getTxStatus(zk, txid); - Thread.sleep(10); - } - // Check that it gets removed - boolean errorSeen = false; - while (!errorSeen) { + // Check that it transitions to SUCCESSFUL and gets removed + final var sawSuccess = new AtomicBoolean(false); + Wait.waitFor(() -> { + TStatus s; try { - s = getTxStatus(zk, txid); - Thread.sleep(10); + switch (s = getTxStatus(zk, txid)) { + case IN_PROGRESS: + if (sawSuccess.get()) { + fail("Should never see IN_PROGRESS after seeing SUCCESSFUL"); + } + break; + case SUCCESSFUL: + // expected, but might be too quick to be detected + if (sawSuccess.compareAndSet(false, true)) { + LOG.debug("Saw expected transaction status change to SUCCESSFUL"); + } + break; + default: + fail("Saw unexpected status: " + s); + } } catch (KeeperException e) { if (e.code() == KeeperException.Code.NONODE) { - errorSeen = true; + if (!sawSuccess.get()) { + LOG.debug("Never saw transaction status change to SUCCESSFUL, but that's okay"); + } + return true; } else { fail("Unexpected error thrown: " + e.getMessage()); } } - } - + // keep waiting for NoNode + return false; + }, SECONDS.toMillis(30), 10); } finally { fate.shutdown(); } From dd2d40ff4171221daf540e0f78bd6e56f878fc06 Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Wed, 27 Nov 2024 22:57:21 -0500 Subject: [PATCH 2/2] Use util code to identify root ZooKeeper path (#5120) Avoid direct use of `Constants.ZROOT + "/" + instanceId` and use the existing `ZooUtil.getRoot(instanceId)` that was made for this purpose instead wherever possible. If a ServerContext is available, use `context.getZooKeeperRoot()` instead. * Use ZooUtil.getRoot() or ServerContext.getZooKeeperRoot() * Use Constants where not currently being used * Remove redundant ZKSecurityTool.getInstancePath * Remove redundant Manager methods that passthrough to ServerContext * Update related tests * Fix use of EasyMock in modified tests: RootTabletLocatorTest and ZookeeperLockCheckerTest * Avoid hard-coded "/accumulo/" in hdfs paths in some ITs that were false-positive potential uses of Constants.ZROOT when I was looking for possibility of replacing literals with constants. For these false-positives, retrieve the actual path from the MiniAccumuloConfig's "instance.volumes" property value, rather than make assumptions about the layout of MiniAccumuloCluster's setup. --- .../core/clientImpl/ClientContext.java | 3 +- .../clientImpl/RootTabletLocatorTest.java | 29 +++++++++--- .../clientImpl/ZookeeperLockCheckerTest.java | 29 +++++++++--- ...AccumuloClusterExistingZooKeepersTest.java | 7 ++- .../apache/accumulo/server/ServerContext.java | 3 +- .../apache/accumulo/server/ServerInfo.java | 3 +- .../server/init/ZooKeeperInitializer.java | 4 +- .../security/handler/ZKPermHandler.java | 7 ++- .../security/handler/ZKSecurityTool.java | 5 -- .../accumulo/server/tables/TableManager.java | 11 +++-- .../server/tablets/UniqueNameAllocator.java | 2 +- .../accumulo/server/util/ChangeSecret.java | 5 +- .../accumulo/server/util/ListInstances.java | 4 +- .../accumulo/server/util/ZooKeeperMain.java | 2 +- .../apache/accumulo/server/util/ZooZap.java | 11 +++-- .../accumulo/server/MockServerContext.java | 8 ++-- .../conf/ServerConfigurationFactoryTest.java | 3 +- .../server/conf/store/PropStoreKeyTest.java | 46 +++++++++--------- .../conf/store/impl/PropStoreEventTest.java | 3 +- .../conf/store/impl/ZooPropStoreTest.java | 3 +- .../ZooAuthenticationKeyWatcherTest.java | 3 +- .../security/handler/ZKAuthenticatorTest.java | 13 +++-- .../accumulo/server/util/AdminTest.java | 47 +++++++++++-------- .../server/util/ServiceStatusCmdTest.java | 3 +- .../apache/accumulo/compactor/Compactor.java | 4 +- .../org/apache/accumulo/manager/Manager.java | 38 ++++++--------- .../manager/ManagerClientServiceHandler.java | 4 +- .../apache/accumulo/manager/ManagerTime.java | 2 +- .../manager/recovery/RecoveryManager.java | 12 ++--- .../accumulo/manager/state/MergeStats.java | 5 +- .../tableOps/compact/CompactRange.java | 8 ++-- .../tableOps/compact/CompactionDriver.java | 8 ++-- .../compact/cancel/CancelCompactions.java | 6 +-- .../tableOps/delete/PreDeleteTable.java | 6 ++- .../PopulateZookeeperWithNamespace.java | 4 +- .../namespace/rename/RenameNamespace.java | 4 +- .../manager/tableOps/rename/RenameTable.java | 4 +- .../manager/tserverOps/ShutdownTServer.java | 7 ++- .../manager/upgrade/Upgrader11to12.java | 2 +- .../compact/CompactionDriverTest.java | 43 +++++++++-------- .../manager/upgrade/Upgrader11to12Test.java | 47 +++++++++---------- .../accumulo/tserver/tablet/Tablet.java | 12 ++--- .../apache/accumulo/test/ExistingMacIT.java | 4 +- .../apache/accumulo/test/ImportExportIT.java | 7 ++- ...hriftServerBindsBeforeZooKeeperLockIT.java | 7 +-- .../apache/accumulo/test/VolumeManagerIT.java | 16 +++++-- .../accumulo/test/fate/zookeeper/FateIT.java | 5 +- .../test/fate/zookeeper/ZooMutatorIT.java | 4 +- .../test/functional/BackupManagerIT.java | 2 +- .../accumulo/test/functional/CloneTestIT.java | 11 +++-- .../test/functional/GarbageCollectorIT.java | 4 +- .../accumulo/test/functional/TableIT.java | 10 ++-- .../accumulo/test/lock/ServiceLockIT.java | 3 +- 53 files changed, 306 insertions(+), 237 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index 298a14329d4..67fec0bfbcc 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@ -510,8 +510,7 @@ public String getRootTabletLocation() { */ public List getManagerLocations() { ensureOpen(); - var zLockManagerPath = - ServiceLock.path(Constants.ZROOT + "/" + getInstanceID() + Constants.ZMANAGER_LOCK); + var zLockManagerPath = ServiceLock.path(getZooKeeperRoot() + Constants.ZMANAGER_LOCK); Timer timer = null; diff --git a/core/src/test/java/org/apache/accumulo/core/clientImpl/RootTabletLocatorTest.java b/core/src/test/java/org/apache/accumulo/core/clientImpl/RootTabletLocatorTest.java index 64f497d0305..0fa52ec2c09 100644 --- a/core/src/test/java/org/apache/accumulo/core/clientImpl/RootTabletLocatorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/clientImpl/RootTabletLocatorTest.java @@ -20,37 +20,54 @@ import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; +import java.util.UUID; + import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.clientImpl.TabletLocatorImpl.TabletServerLockChecker; +import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class RootTabletLocatorTest { + private ClientContext context; private TabletServerLockChecker lockChecker; private ZooCache zc; - private RootTabletLocator rtl; @BeforeEach public void setUp() { - context = createMock(ClientContext.class); - expect(context.getZooKeeperRoot()).andReturn("/accumulo/iid").anyTimes(); + var instanceId = InstanceId.of(UUID.randomUUID()); zc = createMock(ZooCache.class); + context = createMock(ClientContext.class); + expect(context.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(instanceId)).anyTimes(); expect(context.getZooCache()).andReturn(zc).anyTimes(); - replay(context); lockChecker = createMock(TabletServerLockChecker.class); - rtl = new RootTabletLocator(lockChecker); + replay(context, zc, lockChecker); + } + + @AfterEach + public void tearDown() { + verify(context, zc, lockChecker); } @Test public void testInvalidateCache_Server() { + var rtl = new RootTabletLocator(lockChecker); + + verify(zc); + reset(zc); zc.clear(context.getZooKeeperRoot() + Constants.ZTSERVERS + "/server"); + expectLastCall().once(); replay(zc); + rtl.invalidateCache(context, "server"); - verify(zc); } } diff --git a/core/src/test/java/org/apache/accumulo/core/clientImpl/ZookeeperLockCheckerTest.java b/core/src/test/java/org/apache/accumulo/core/clientImpl/ZookeeperLockCheckerTest.java index 7776a55aa1f..991e4d2dba9 100644 --- a/core/src/test/java/org/apache/accumulo/core/clientImpl/ZookeeperLockCheckerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/clientImpl/ZookeeperLockCheckerTest.java @@ -20,34 +20,51 @@ import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; +import java.util.UUID; + import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class ZookeeperLockCheckerTest { + private ClientContext context; private ZooCache zc; - private ZookeeperLockChecker zklc; @BeforeEach public void setUp() { - context = createMock(ClientContext.class); - expect(context.getZooKeeperRoot()).andReturn("/accumulo/iid").anyTimes(); + var instanceId = InstanceId.of(UUID.randomUUID()); zc = createMock(ZooCache.class); + context = createMock(ClientContext.class); + expect(context.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(instanceId)).anyTimes(); expect(context.getZooCache()).andReturn(zc).anyTimes(); - replay(context); - zklc = new ZookeeperLockChecker(context); + replay(context, zc); + } + + @AfterEach + public void tearDown() { + verify(context, zc); } @Test public void testInvalidateCache() { + var zklc = new ZookeeperLockChecker(context); + + verify(zc); + reset(zc); zc.clear(context.getZooKeeperRoot() + Constants.ZTSERVERS + "/server"); + expectLastCall().once(); replay(zc); + zklc.invalidateCache("server"); - verify(zc); } } diff --git a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterExistingZooKeepersTest.java b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterExistingZooKeepersTest.java index c428127b58d..575588993fd 100644 --- a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterExistingZooKeepersTest.java +++ b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterExistingZooKeepersTest.java @@ -26,8 +26,10 @@ import java.io.File; import java.util.Map; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.commons.io.FileUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -74,8 +76,9 @@ public void canConnectViaExistingZooKeeper() throws Exception { Map tableIds = client.tableOperations().tableIdMap(); assertTrue(tableIds.containsKey(tableName)); - String zkTablePath = String.format("/accumulo/%s/tables/%s/name", - client.instanceOperations().getInstanceId().canonical(), tableIds.get(tableName)); + String zkTablePath = String.format("%s%s/%s/name", + ZooUtil.getRoot(client.instanceOperations().getInstanceId()), Constants.ZTABLES, + tableIds.get(tableName)); try (CuratorFramework curatorClient = CuratorFrameworkFactory.newClient(zooKeeper.getConnectString(), new RetryOneTime(1))) { curatorClient.start(); diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index bf758218288..a20b5c62bbd 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -52,6 +52,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.rpc.SslConnectionParams; @@ -118,7 +119,7 @@ private ServerContext(ServerInfo info) { serverDirs = info.getServerDirs(); propStore = memoize(() -> ZooPropStore.initialize(getInstanceID(), getZooReaderWriter())); - zkUserPath = memoize(() -> Constants.ZROOT + "/" + getInstanceID() + Constants.ZUSERS); + zkUserPath = memoize(() -> ZooUtil.getRoot(getInstanceID()) + Constants.ZUSERS); tableManager = memoize(() -> new TableManager(this)); nameAllocator = memoize(() -> new UniqueNameAllocator(this)); diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java b/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java index f0fd0c83947..aa7d2149dde 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerInfo.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.fate.zookeeper.ZooCacheFactory; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.singletons.SingletonManager; import org.apache.accumulo.core.singletons.SingletonManager.Mode; import org.apache.accumulo.server.fs.VolumeManager; @@ -79,7 +80,7 @@ public class ServerInfo implements ClientInfo { + "Run \"accumulo org.apache.accumulo.server.util.ListInstances\" to see a list."); } instanceID = InstanceId.of(new String(iidb, UTF_8)); - if (zooCache.get(Constants.ZROOT + "/" + instanceID) == null) { + if (zooCache.get(ZooUtil.getRoot(instanceID)) == null) { if (instanceName == null) { throw new IllegalStateException( "Instance id " + instanceID + " does not exist in zookeeper"); diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java index 1fc771dc097..8a78f6c2546 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java @@ -71,7 +71,7 @@ void initializeConfig(final InstanceId instanceId, final ZooReaderWriter zoo) { zoo.putPersistentData(Constants.ZROOT, new byte[0], ZooUtil.NodeExistsPolicy.SKIP, ZooDefs.Ids.OPEN_ACL_UNSAFE); - String zkInstanceRoot = Constants.ZROOT + "/" + instanceId; + String zkInstanceRoot = ZooUtil.getRoot(instanceId); zoo.putPersistentData(zkInstanceRoot, EMPTY_BYTE_ARRAY, ZooUtil.NodeExistsPolicy.SKIP); var sysPropPath = SystemPropKey.of(instanceId).getPath(); VersionedProperties vProps = new VersionedProperties(); @@ -109,7 +109,7 @@ void initialize(final ServerContext context, final boolean clearInstanceName, ZooUtil.NodeExistsPolicy.FAIL); // setup the instance - String zkInstanceRoot = Constants.ZROOT + "/" + instanceId; + String zkInstanceRoot = context.getZooKeeperRoot(); zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES, Constants.ZTABLES_INITIAL_ID, ZooUtil.NodeExistsPolicy.FAIL); zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java index 750c7d15f93..f5b3768a854 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKPermHandler.java @@ -27,12 +27,12 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.NamespaceNotFoundException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.clientImpl.Namespace; import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; @@ -66,10 +66,9 @@ public class ZKPermHandler implements PermissionHandler { public void initialize(ServerContext context) { zooCache = new ZooCache(context.getZooReader(), null); zoo = context.getZooReaderWriter(); - InstanceId instanceId = context.getInstanceID(); zkUserPath = context.zkUserPath(); - ZKTablePath = ZKSecurityTool.getInstancePath(instanceId) + "/tables"; - ZKNamespacePath = ZKSecurityTool.getInstancePath(instanceId) + "/namespaces"; + ZKTablePath = context.getZooKeeperRoot() + Constants.ZTABLES; + ZKNamespacePath = context.getZooKeeperRoot() + Constants.ZNAMESPACES; } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java index af554f06f2e..ecbb1cd31b2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/handler/ZKSecurityTool.java @@ -31,9 +31,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.NamespacePermission; import org.apache.accumulo.core.security.SystemPermission; @@ -191,7 +189,4 @@ public static Set convertNamespacePermissions(byte[] namesp return toReturn; } - public static String getInstancePath(InstanceId instanceId) { - return Constants.ZROOT + "/" + instanceId; - } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java index 15ce8d59419..bd8f74e6f0c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.core.manager.state.tables.TableState; @@ -77,7 +78,7 @@ public static void prepareNewNamespaceState(final ServerContext context, Namespa final InstanceId instanceId = context.getInstanceID(); log.debug("Creating ZooKeeper entries for new namespace {} (ID: {})", namespace, namespaceId); context.getZooReaderWriter().putPersistentData( - Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId, new byte[0], + context.getZooKeeperRoot() + Constants.ZNAMESPACES + "/" + namespaceId, new byte[0], existsPolicy); var propKey = NamespacePropKey.of(instanceId, namespaceId); if (!propStore.exists(propKey)) { @@ -94,7 +95,7 @@ public static void prepareNewTableState(ZooReaderWriter zoo, PropStore propStore tableName, tableId, namespaceId); Pair qualifiedTableName = TableNameUtil.qualify(tableName); tableName = qualifiedTableName.getSecond(); - String zTablePath = Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId; + String zTablePath = ZooUtil.getRoot(instanceId) + Constants.ZTABLES + "/" + tableId; zoo.putPersistentData(zTablePath, new byte[0], existsPolicy); zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAMESPACE, namespaceId.canonical().getBytes(UTF_8), existsPolicy); @@ -220,10 +221,10 @@ public void cloneTable(TableId srcTableId, TableId tableId, String tableName, prepareNewTableState(zoo, context.getPropStore(), instanceID, tableId, namespaceId, tableName, TableState.NEW, NodeExistsPolicy.OVERWRITE); - String srcTablePath = Constants.ZROOT + "/" + instanceID + Constants.ZTABLES + "/" + srcTableId - + Constants.ZCONFIG; + String srcTablePath = + context.getZooKeeperRoot() + Constants.ZTABLES + "/" + srcTableId + Constants.ZCONFIG; String newTablePath = - Constants.ZROOT + "/" + instanceID + Constants.ZTABLES + "/" + tableId + Constants.ZCONFIG; + context.getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + Constants.ZCONFIG; zoo.recursiveCopyPersistentOverwrite(srcTablePath, newTablePath); PropUtil.setProperties(context, TablePropKey.of(context, tableId), propertiesToSet); diff --git a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java index 14026124df3..7aead5026ec 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java @@ -50,7 +50,7 @@ public class UniqueNameAllocator { public UniqueNameAllocator(ServerContext context) { this.context = context; - nextNamePath = Constants.ZROOT + "/" + context.getInstanceID() + Constants.ZNEXT_FILE; + nextNamePath = context.getZooKeeperRoot() + Constants.ZNEXT_FILE; } public synchronized String getNextName() { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java index 094167e0ceb..5a794c3eebe 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.UUID; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; @@ -145,7 +146,7 @@ private static void rewriteZooKeeperInstance(final ServerContext context, } } }); - String path = "/accumulo/instances/" + context.getInstanceName(); + String path = Constants.ZROOT + Constants.ZINSTANCES + "/" + context.getInstanceName(); orig.recursiveDelete(path, NodeMissingPolicy.SKIP); new_.putPersistentData(path, newInstanceId.canonical().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); @@ -201,6 +202,6 @@ private static void checkHdfsAccessPermissions(FileStatus stat, FsAction mode) t private static void deleteInstance(ServerContext context, String oldPass) throws Exception { ZooReaderWriter orig = context.getZooReader().asWriter(oldPass); - orig.recursiveDelete("/accumulo/" + context.getInstanceID(), NodeMissingPolicy.SKIP); + orig.recursiveDelete(context.getZooKeeperRoot(), NodeMissingPolicy.SKIP); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java index 275e11e5a06..24e7b218b73 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListInstances.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.fate.zookeeper.ZooReader; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.lock.ServiceLockData; import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; @@ -165,8 +166,7 @@ private static String getManager(ZooCache cache, InstanceId iid, boolean printEr } try { - var zLockManagerPath = - ServiceLock.path(Constants.ZROOT + "/" + iid + Constants.ZMANAGER_LOCK); + var zLockManagerPath = ServiceLock.path(ZooUtil.getRoot(iid) + Constants.ZMANAGER_LOCK); Optional sld = ServiceLock.getLockData(cache, zLockManagerPath, null); if (sld.isEmpty()) { return null; diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java index b98ef90b340..b7cb4d5732e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooKeeperMain.java @@ -69,7 +69,7 @@ public void execute(final String[] args) throws Exception { } System.out.println("The accumulo instance id is " + context.getInstanceID()); if (!opts.servers.contains("/")) { - opts.servers += "/accumulo/" + context.getInstanceID(); + opts.servers += context.getZooKeeperRoot(); } org.apache.zookeeper.ZooKeeperMain .main(new String[] {"-server", opts.servers, "-timeout", "" + (opts.timeout * 1000)}); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java index 3a2b6179841..87c1eacf0ee 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ZooZap.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.singletons.SingletonManager; @@ -107,7 +108,7 @@ public void execute(String[] args) throws Exception { ZooReaderWriter zoo = new ZooReaderWriter(siteConf); if (opts.zapManager) { - String managerLockPath = Constants.ZROOT + "/" + iid + Constants.ZMANAGER_LOCK; + String managerLockPath = ZooUtil.getRoot(iid) + Constants.ZMANAGER_LOCK; try { zapDirectory(zoo, managerLockPath, opts); @@ -117,7 +118,7 @@ public void execute(String[] args) throws Exception { } if (opts.zapTservers) { - String tserversPath = Constants.ZROOT + "/" + iid + Constants.ZTSERVERS; + String tserversPath = ZooUtil.getRoot(iid) + Constants.ZTSERVERS; try { List children = zoo.getChildren(tserversPath); for (String child : children) { @@ -140,7 +141,7 @@ public void execute(String[] args) throws Exception { } if (opts.zapCoordinators) { - final String coordinatorPath = Constants.ZROOT + "/" + iid + Constants.ZCOORDINATOR_LOCK; + final String coordinatorPath = ZooUtil.getRoot(iid) + Constants.ZCOORDINATOR_LOCK; try { if (zoo.exists(coordinatorPath)) { zapDirectory(zoo, coordinatorPath, opts); @@ -151,7 +152,7 @@ public void execute(String[] args) throws Exception { } if (opts.zapCompactors) { - String compactorsBasepath = Constants.ZROOT + "/" + iid + Constants.ZCOMPACTORS; + String compactorsBasepath = ZooUtil.getRoot(iid) + Constants.ZCOMPACTORS; try { if (zoo.exists(compactorsBasepath)) { List queues = zoo.getChildren(compactorsBasepath); @@ -167,7 +168,7 @@ public void execute(String[] args) throws Exception { } if (opts.zapScanServers) { - String sserversPath = Constants.ZROOT + "/" + iid + Constants.ZSSERVERS; + String sserversPath = ZooUtil.getRoot(iid) + Constants.ZSSERVERS; try { if (zoo.exists(sserversPath)) { List children = zoo.getChildren(sserversPath); diff --git a/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java b/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java index 801e0c301ca..338acf94860 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java +++ b/server/base/src/test/java/org/apache/accumulo/server/MockServerContext.java @@ -23,11 +23,13 @@ import java.util.Properties; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.conf.store.PropStore; import org.easymock.EasyMock; @@ -47,9 +49,9 @@ public static ServerContext get() { public static ServerContext getWithZK(InstanceId instanceID, String zk, int zkTimeout) { var sc = get(); - expect(sc.getZooKeeperRoot()).andReturn("/accumulo/" + instanceID).anyTimes(); + expect(sc.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(instanceID)).anyTimes(); expect(sc.getInstanceID()).andReturn(instanceID).anyTimes(); - expect(sc.zkUserPath()).andReturn("/accumulo/" + instanceID + "/users").anyTimes(); + expect(sc.zkUserPath()).andReturn(ZooUtil.getRoot(instanceID) + Constants.ZUSERS).anyTimes(); expect(sc.getZooKeepers()).andReturn(zk).anyTimes(); expect(sc.getZooKeepersSessionTimeOut()).andReturn(zkTimeout).anyTimes(); return sc; @@ -61,7 +63,7 @@ public static ServerContext getMockContextWithPropStore(final InstanceId instanc ServerContext sc = createMock(ServerContext.class); expect(sc.getInstanceID()).andReturn(instanceID).anyTimes(); expect(sc.getZooReaderWriter()).andReturn(zrw).anyTimes(); - expect(sc.getZooKeeperRoot()).andReturn("/accumulo/" + instanceID).anyTimes(); + expect(sc.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(instanceID)).anyTimes(); expect(sc.getPropStore()).andReturn(propStore).anyTimes(); return sc; } diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java index 71e13cda8a2..2743b2a7f28 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/conf/ServerConfigurationFactoryTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.codec.VersionedProperties; import org.apache.accumulo.server.conf.store.NamespacePropKey; @@ -79,7 +80,7 @@ public void setUp() { expectLastCall().anyTimes(); context = createMock(ServerContext.class); - expect(context.getZooKeeperRoot()).andReturn("/accumulo/" + IID).anyTimes(); + expect(context.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(IID)).anyTimes(); expect(context.getInstanceID()).andReturn(IID).anyTimes(); expect(context.getZooKeepers()).andReturn(ZK_HOST).anyTimes(); expect(context.getZooKeepersSessionTimeOut()).andReturn(ZK_TIMEOUT).anyTimes(); diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/PropStoreKeyTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/PropStoreKeyTest.java index f3d040275d4..80abef3aa77 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/PropStoreKeyTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/PropStoreKeyTest.java @@ -20,6 +20,7 @@ import static org.apache.accumulo.core.Constants.ZCONFIG; import static org.apache.accumulo.core.Constants.ZNAMESPACES; +import static org.apache.accumulo.core.Constants.ZROOT; import static org.apache.accumulo.core.Constants.ZTABLES; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; @@ -35,6 +36,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.ServerContext; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -93,48 +95,46 @@ public void tableType() { @Test public void fromPathTest() { - - var iid = "3f9976c6-3bf1-41ab-9751-1b0a9be3551d"; - - PropStoreKey t1 = PropStoreKey.fromPath("/accumulo/" + iid + "/tables/t1" + ZCONFIG); + var t1 = PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZTABLES + "/t1" + ZCONFIG); assertNotNull(t1); assertEquals(TableId.of("t1"), t1.getId()); - PropStoreKey n1 = PropStoreKey.fromPath("/accumulo/" + iid + "/namespaces/n1" + ZCONFIG); + var n1 = PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZNAMESPACES + "/n1" + ZCONFIG); assertNotNull(n1); assertEquals(NamespaceId.of("n1"), n1.getId()); assertNotNull(n1.getId()); - PropStoreKey s1 = PropStoreKey.fromPath("/accumulo/" + iid + ZCONFIG); + var s1 = PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZCONFIG); assertNotNull(s1); // system config returns instance id as id placeholder - assertEquals(iid, s1.getId().canonical()); + assertEquals(instanceId, s1.getId()); } @Test public void invalidKeysTest() { - var iid = "3f9976c6-3bf1-41ab-9751-1b0a9be3551d"; - // too short - assertNull(PropStoreKey.fromPath("/accumulo")); + assertNull(PropStoreKey.fromPath(ZROOT)); // not a system config - assertTrue(PropStoreKey.fromPath("/accumulo/" + iid + ZCONFIG) instanceof SystemPropKey); + assertTrue( + PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZCONFIG) instanceof SystemPropKey); assertNull(PropStoreKey.fromPath("/foo")); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + "/foo")); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + ZCONFIG + "/foo")); - - assertTrue(PropStoreKey - .fromPath("/accumulo/" + iid + ZTABLES + "/a" + ZCONFIG) instanceof TablePropKey); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + ZTABLES + ZCONFIG)); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + "/invalid/a" + ZCONFIG)); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + ZTABLES + "/a" + ZCONFIG + "/foo")); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + "/foo")); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZCONFIG + "/foo")); assertTrue(PropStoreKey - .fromPath("/accumulo/" + iid + ZNAMESPACES + "/a" + ZCONFIG) instanceof NamespacePropKey); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + ZNAMESPACES + ZCONFIG)); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + "/invalid/a" + ZCONFIG)); - assertNull(PropStoreKey.fromPath("/accumulo/" + iid + ZNAMESPACES + "/a" + ZCONFIG + "/foo")); + .fromPath(ZooUtil.getRoot(instanceId) + ZTABLES + "/a" + ZCONFIG) instanceof TablePropKey); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZTABLES + ZCONFIG)); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + "/invalid/a" + ZCONFIG)); + assertNull( + PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZTABLES + "/a" + ZCONFIG + "/foo")); + + assertTrue(PropStoreKey.fromPath( + ZooUtil.getRoot(instanceId) + ZNAMESPACES + "/a" + ZCONFIG) instanceof NamespacePropKey); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZNAMESPACES + ZCONFIG)); + assertNull(PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + "/invalid/a" + ZCONFIG)); + assertNull( + PropStoreKey.fromPath(ZooUtil.getRoot(instanceId) + ZNAMESPACES + "/a" + ZCONFIG + "/foo")); } @Test diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java index 71247b4d3b5..ca67be50fb4 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/PropStoreEventTest.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.codec.VersionedPropCodec; import org.apache.accumulo.server.conf.codec.VersionedProperties; @@ -72,7 +73,7 @@ public void initCommonMocks() throws Exception { expect(context.getZooKeepersSessionTimeOut()).andReturn(500).anyTimes(); expect(context.getInstanceID()).andReturn(instanceId).anyTimes(); - expect(zrw.exists(eq("/accumulo/" + instanceId), anyObject())).andReturn(true).anyTimes(); + expect(zrw.exists(eq(ZooUtil.getRoot(instanceId)), anyObject())).andReturn(true).anyTimes(); readyMonitor = createMock(ReadyMonitor.class); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropStoreTest.java b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropStoreTest.java index 11d9f9e4714..a09f2bef8b5 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropStoreTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/conf/store/impl/ZooPropStoreTest.java @@ -45,6 +45,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.codec.VersionedPropCodec; import org.apache.accumulo.server.conf.codec.VersionedProperties; @@ -75,7 +76,7 @@ public void init() throws Exception { expect(zrw.getSessionTimeout()).andReturn(2_000).anyTimes(); expect(context.getInstanceID()).andReturn(instanceId).anyTimes(); - expect(zrw.exists(eq("/accumulo/" + instanceId), anyObject())).andReturn(true).anyTimes(); + expect(zrw.exists(eq(ZooUtil.getRoot(instanceId)), anyObject())).andReturn(true).anyTimes(); } @AfterEach diff --git a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java index 01bc04b3a87..df71d4fe8c0 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java @@ -42,6 +42,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher.Event.EventType; @@ -75,7 +76,7 @@ public static void setupKeyGenerator() throws Exception { public void setupMocks() { zk = createMock(ZooReader.class); instanceId = InstanceId.of(UUID.randomUUID()); - baseNode = "/accumulo/" + instanceId + Constants.ZDELEGATION_TOKEN_KEYS; + baseNode = ZooUtil.getRoot(instanceId) + Constants.ZDELEGATION_TOKEN_KEYS; secretManager = new AuthenticationTokenSecretManager(instanceId, tokenLifetime); keyWatcher = new ZooAuthenticationKeyWatcher(secretManager, zk, baseNode); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/security/handler/ZKAuthenticatorTest.java b/server/base/src/test/java/org/apache/accumulo/server/security/handler/ZKAuthenticatorTest.java index 6616b052503..6621feb93ff 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/security/handler/ZKAuthenticatorTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/handler/ZKAuthenticatorTest.java @@ -35,10 +35,12 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SystemPermission; import org.apache.accumulo.core.security.TablePermission; @@ -140,16 +142,17 @@ public void testUserAuthentication() throws Exception { byte[] newHash = ZKSecurityTool.createPass(rawPass.clone()); // mocking zk interaction - ServerContext context = MockServerContext.getWithZK(InstanceId.of("example"), "", 30_000); + var instanceId = InstanceId.of("example"); + ServerContext context = MockServerContext.getWithZK(instanceId, "", 30_000); ZooReaderWriter zr = createMock(ZooReaderWriter.class); expect(context.getZooReader()).andReturn(zr).anyTimes(); ZooKeeper zk = createMock(ZooKeeper.class); expect(zk.getChildren(anyObject(), anyObject())).andReturn(Arrays.asList(principal)).anyTimes(); - expect(zk.exists(matches("/accumulo/example/users/" + principal), anyObject(Watcher.class))) - .andReturn(new Stat()).anyTimes(); + expect(zk.exists(matches(ZooUtil.getRoot(instanceId) + Constants.ZUSERS + "/" + principal), + anyObject(Watcher.class))).andReturn(new Stat()).anyTimes(); expect(zr.getZooKeeper()).andReturn(zk).anyTimes(); - expect(zk.getData(matches("/accumulo/example/users/" + principal), anyObject(), anyObject())) - .andReturn(newHash).once(); + expect(zk.getData(matches(ZooUtil.getRoot(instanceId) + Constants.ZUSERS + "/" + principal), + anyObject(), anyObject())).andReturn(newHash).once(); replay(context, zr, zk); // creating authenticator diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java index 2e6754176e9..18a90454028 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java @@ -18,6 +18,13 @@ */ package org.apache.accumulo.server.util; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.getCurrentArguments; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Collections; @@ -28,69 +35,71 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.fate.zookeeper.ZooCache.ZcStat; -import org.easymock.EasyMock; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.junit.jupiter.api.Test; public class AdminTest { @Test public void testZooKeeperTserverPath() { - ClientContext context = EasyMock.createMock(ClientContext.class); + ClientContext context = createMock(ClientContext.class); InstanceId instanceId = InstanceId.of(UUID.randomUUID()); - EasyMock.expect(context.getZooKeeperRoot()).andReturn(Constants.ZROOT + "/" + instanceId); + expect(context.getZooKeeperRoot()).andReturn(ZooUtil.getRoot(instanceId)); - EasyMock.replay(context); + replay(context); - assertEquals(Constants.ZROOT + "/" + instanceId + Constants.ZTSERVERS, + assertEquals(ZooUtil.getRoot(instanceId) + Constants.ZTSERVERS, Admin.getTServersZkPath(context)); - EasyMock.verify(context); + verify(context); } @Test public void testQualifySessionId() { - ZooCache zc = EasyMock.createMock(ZooCache.class); + ZooCache zc = createMock(ZooCache.class); + InstanceId instanceId = InstanceId.of(UUID.randomUUID()); - String root = "/accumulo/id/tservers"; + String root = ZooUtil.getRoot(instanceId) + Constants.ZTSERVERS; String server = "localhost:12345"; final long session = 123456789L; String serverPath = root + "/" + server; String validZLockEphemeralNode = "zlock#" + UUID.randomUUID() + "#0000000000"; - EasyMock.expect(zc.getChildren(serverPath)) + expect(zc.getChildren(serverPath)) .andReturn(Collections.singletonList(validZLockEphemeralNode)); - EasyMock.expect(zc.get(EasyMock.eq(serverPath + "/" + validZLockEphemeralNode), - EasyMock.anyObject(ZcStat.class))).andAnswer(() -> { - ZcStat stat = (ZcStat) EasyMock.getCurrentArguments()[1]; + expect(zc.get(eq(serverPath + "/" + validZLockEphemeralNode), anyObject(ZcStat.class))) + .andAnswer(() -> { + ZcStat stat = (ZcStat) getCurrentArguments()[1]; stat.setEphemeralOwner(session); return new byte[0]; }); - EasyMock.replay(zc); + replay(zc); assertEquals(server + "[" + Long.toHexString(session) + "]", Admin.qualifyWithZooKeeperSessionId(root, zc, server)); - EasyMock.verify(zc); + verify(zc); } @Test public void testCannotQualifySessionId() { - ZooCache zc = EasyMock.createMock(ZooCache.class); + ZooCache zc = createMock(ZooCache.class); + InstanceId instanceId = InstanceId.of(UUID.randomUUID()); - String root = "/accumulo/id/tservers"; + String root = ZooUtil.getRoot(instanceId) + Constants.ZTSERVERS; String server = "localhost:12345"; String serverPath = root + "/" + server; - EasyMock.expect(zc.getChildren(serverPath)).andReturn(Collections.emptyList()); + expect(zc.getChildren(serverPath)).andReturn(Collections.emptyList()); - EasyMock.replay(zc); + replay(zc); // A server that isn't in ZooKeeper. Can't qualify it, should return the original assertEquals(server, Admin.qualifyWithZooKeeperSessionId(root, zc, server)); - EasyMock.verify(zc); + verify(zc); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java index b48440b666e..5a8bb11b775 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java @@ -38,6 +38,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.util.serviceStatus.ServiceStatusReport; import org.apache.accumulo.server.util.serviceStatus.StatusSummary; @@ -59,7 +60,7 @@ public class ServiceStatusCmdTest { @BeforeEach public void populateContext() { InstanceId iid = InstanceId.of(UUID.randomUUID()); - zRoot = "/accumulo/" + iid.canonical(); + zRoot = ZooUtil.getRoot(iid); context = createMock(ServerContext.class); expect(context.getInstanceID()).andReturn(iid).anyTimes(); expect(context.getZooKeeperRoot()).andReturn(zRoot).anyTimes(); diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index c93a8cb8caa..c8908900b98 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -217,8 +217,8 @@ protected void checkIfCanceled() { } if (job.getKind() == TCompactionKind.USER) { - String zTablePath = Constants.ZROOT + "/" + getContext().getInstanceID() - + Constants.ZTABLES + "/" + extent.tableId() + Constants.ZTABLE_COMPACT_CANCEL_ID; + String zTablePath = getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + extent.tableId() + Constants.ZTABLE_COMPACT_CANCEL_ID; byte[] id = getContext().getZooCache().get(zTablePath); if (id == null) { // table probably deleted diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 0a598712ada..ea5bb097dca 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -63,7 +63,6 @@ import org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.AgeOffStore; @@ -431,7 +430,7 @@ protected Manager(ConfigOpts opts, String[] args) throws IOException { AccumuloConfiguration aconf = context.getConfiguration(); log.info("Version {}", Constants.VERSION); - log.info("Instance {}", getInstanceID()); + log.info("Instance {}", context.getInstanceID()); timeKeeper = new ManagerTime(this, aconf); tserverSet = new LiveTServerSet(context, this); initializeBalancer(); @@ -449,7 +448,7 @@ protected Manager(ConfigOpts opts, String[] args) throws IOException { final long tokenUpdateInterval = aconf.getTimeInMillis(Property.GENERAL_DELEGATION_TOKEN_UPDATE_INTERVAL); keyDistributor = new ZooAuthenticationKeyDistributor(context.getZooReaderWriter(), - getZooKeeperRoot() + Constants.ZDELEGATION_TOKEN_KEYS); + context.getZooKeeperRoot() + Constants.ZDELEGATION_TOKEN_KEYS); authenticationTokenKeyManager = new AuthenticationTokenKeyManager(context.getSecretManager(), keyDistributor, tokenUpdateInterval, tokenLifetime); delegationTokensAvailable = true; @@ -461,14 +460,6 @@ protected Manager(ConfigOpts opts, String[] args) throws IOException { aconf.getTimeInMillis(Property.MANAGER_RECOVERY_WAL_EXISTENCE_CACHE_TIME); } - public InstanceId getInstanceID() { - return getContext().getInstanceID(); - } - - public String getZooKeeperRoot() { - return getContext().getZooKeeperRoot(); - } - public TServerConnection getConnection(TServerInstance server) { return tserverSet.getConnection(server); } @@ -477,7 +468,7 @@ public MergeInfo getMergeInfo(TableId tableId) { ServerContext context = getContext(); synchronized (mergeLock) { try { - String path = getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + "/merge"; + String path = context.getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + "/merge"; if (!context.getZooReaderWriter().exists(path)) { return new MergeInfo(); } @@ -501,8 +492,8 @@ public void setMergeState(MergeInfo info, MergeState state) throws KeeperException, InterruptedException { ServerContext context = getContext(); synchronized (mergeLock) { - String path = - getZooKeeperRoot() + Constants.ZTABLES + "/" + info.getExtent().tableId() + "/merge"; + String path = context.getZooKeeperRoot() + Constants.ZTABLES + "/" + + info.getExtent().tableId() + "/merge"; info.setState(state); if (state.equals(MergeState.NONE)) { context.getZooReaderWriter().recursiveDelete(path, NodeMissingPolicy.SKIP); @@ -524,7 +515,7 @@ public void setMergeState(MergeInfo info, MergeState state) public void clearMergeState(TableId tableId) throws KeeperException, InterruptedException { synchronized (mergeLock) { - String path = getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + "/merge"; + String path = getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + "/merge"; getContext().getZooReaderWriter().recursiveDelete(path, NodeMissingPolicy.SKIP); mergeLock.notifyAll(); } @@ -534,8 +525,8 @@ public void clearMergeState(TableId tableId) throws KeeperException, Interrupted void setManagerGoalState(ManagerGoalState state) { try { getContext().getZooReaderWriter().putPersistentData( - getZooKeeperRoot() + Constants.ZMANAGER_GOAL_STATE, state.name().getBytes(UTF_8), - NodeExistsPolicy.OVERWRITE); + getContext().getZooKeeperRoot() + Constants.ZMANAGER_GOAL_STATE, + state.name().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); } catch (Exception ex) { log.error("Unable to set manager goal state in zookeeper"); } @@ -545,7 +536,7 @@ ManagerGoalState getManagerGoalState() { while (true) { try { byte[] data = getContext().getZooReaderWriter() - .getData(getZooKeeperRoot() + Constants.ZMANAGER_GOAL_STATE); + .getData(getContext().getZooKeeperRoot() + Constants.ZMANAGER_GOAL_STATE); return ManagerGoalState.valueOf(new String(data, UTF_8)); } catch (Exception e) { log.error("Problem getting real goal state from zookeeper: ", e); @@ -1215,7 +1206,7 @@ private List checkMigrationSanity(Set current, @Override public void run() { final ServerContext context = getContext(); - final String zroot = getZooKeeperRoot(); + final String zroot = context.getZooKeeperRoot(); // ACCUMULO-4424 Put up the Thrift servers before getting the lock as a sign of process health // when a hot-standby @@ -1347,10 +1338,11 @@ boolean canSuspendTablets() { throw new IllegalStateException("Upgrade coordinator is unexpectedly not complete"); } try { - final AgeOffStore store = new AgeOffStore<>( - new org.apache.accumulo.core.fate.ZooStore<>(getZooKeeperRoot() + Constants.ZFATE, - context.getZooReaderWriter()), - HOURS.toMillis(8), System::currentTimeMillis); + final AgeOffStore store = + new AgeOffStore<>( + new org.apache.accumulo.core.fate.ZooStore<>( + context.getZooKeeperRoot() + Constants.ZFATE, context.getZooReaderWriter()), + HOURS.toMillis(8), System::currentTimeMillis); Fate f = initializeFateInstance(store, getConfiguration()); fateRef.set(f); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java index ad00a842eab..02057c3b897 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java @@ -106,8 +106,8 @@ public long initiateFlush(TInfo tinfo, TCredentials c, String tableIdStr) throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); } - String zTablePath = Constants.ZROOT + "/" + manager.getInstanceID() + Constants.ZTABLES + "/" - + tableId + Constants.ZTABLE_FLUSH_ID; + String zTablePath = manager.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + + Constants.ZTABLE_FLUSH_ID; ZooReaderWriter zoo = manager.getContext().getZooReaderWriter(); byte[] fid; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java index fa7020a0ef7..b7cb2bfcdcb 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java @@ -89,7 +89,7 @@ public class ManagerTime { private final AtomicReference skewAmount; public ManagerTime(Manager manager, AccumuloConfiguration conf) throws IOException { - this.zPath = manager.getZooKeeperRoot() + Constants.ZMANAGER_TICK; + this.zPath = manager.getContext().getZooKeeperRoot() + Constants.ZMANAGER_TICK; this.zk = manager.getContext().getZooReaderWriter(); this.manager = manager; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 5d48d839a00..7efd6f45118 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -78,7 +78,7 @@ public RecoveryManager(Manager manager, long timeToCacheExistsInMillis) { zooCache = new ZooCache(manager.getContext().getZooReader(), null); try { List workIDs = - new DistributedWorkQueue(manager.getZooKeeperRoot() + Constants.ZRECOVERY, + new DistributedWorkQueue(manager.getContext().getZooKeeperRoot() + Constants.ZRECOVERY, manager.getConfiguration(), manager.getContext()).getWorkQueued(); sortsQueued.addAll(workIDs); } catch (Exception e) { @@ -131,14 +131,15 @@ public void run() { private void initiateSort(String sortId, String source, final String destination) throws KeeperException, InterruptedException { String work = source + "|" + destination; - new DistributedWorkQueue(manager.getZooKeeperRoot() + Constants.ZRECOVERY, + new DistributedWorkQueue(manager.getContext().getZooKeeperRoot() + Constants.ZRECOVERY, manager.getConfiguration(), manager.getContext()).addWork(sortId, work.getBytes(UTF_8)); synchronized (this) { sortsQueued.add(sortId); } - final String path = manager.getZooKeeperRoot() + Constants.ZRECOVERY + "/" + sortId; + final String path = + manager.getContext().getZooKeeperRoot() + Constants.ZRECOVERY + "/" + sortId; log.info("Created zookeeper entry {} with data {}", path, work); } @@ -180,9 +181,8 @@ public boolean recoverLogs(KeyExtent extent, Collection walogs) throws sortQueued = sortsQueued.contains(sortId); } - if (sortQueued - && zooCache.get(manager.getZooKeeperRoot() + Constants.ZRECOVERY + "/" + sortId) - == null) { + if (sortQueued && zooCache.get( + manager.getContext().getZooKeeperRoot() + Constants.ZRECOVERY + "/" + sortId) == null) { synchronized (this) { sortsQueued.remove(sortId); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/state/MergeStats.java b/server/manager/src/main/java/org/apache/accumulo/manager/state/MergeStats.java index a60f8f46dce..c414cff441a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/state/MergeStats.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/state/MergeStats.java @@ -33,7 +33,6 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; -import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.TabletLocationState; import org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException; @@ -238,8 +237,8 @@ public static void main(String[] args) throws Exception { ZooReaderWriter zooReaderWriter = opts.getServerContext().getZooReaderWriter(); for (Entry entry : tableIdMap.entrySet()) { final String table = entry.getKey(), tableId = entry.getValue(); - String path = ZooUtil.getRoot(client.instanceOperations().getInstanceId()) - + Constants.ZTABLES + "/" + tableId + "/merge"; + String path = opts.getServerContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + tableId + "/merge"; MergeInfo info = new MergeInfo(); if (zooReaderWriter.exists(path)) { byte[] data = zooReaderWriter.getData(path); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java index 46e0ce12c52..4c2fcb06457 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactRange.java @@ -97,8 +97,8 @@ public long isReady(long tid, Manager env) throws Exception { @Override public Repo call(final long tid, Manager env) throws Exception { - String zTablePath = Constants.ZROOT + "/" + env.getInstanceID() + Constants.ZTABLES + "/" - + tableId + Constants.ZTABLE_COMPACT_ID; + String zTablePath = env.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + + Constants.ZTABLE_COMPACT_ID; ZooReaderWriter zoo = env.getContext().getZooReaderWriter(); byte[] cid; @@ -147,8 +147,8 @@ public Repo call(final long tid, Manager env) throws Exception { static void removeIterators(Manager environment, final long txid, TableId tableId) throws Exception { - String zTablePath = Constants.ZROOT + "/" + environment.getInstanceID() + Constants.ZTABLES - + "/" + tableId + Constants.ZTABLE_COMPACT_ID; + String zTablePath = environment.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + tableId + Constants.ZTABLE_COMPACT_ID; ZooReaderWriter zoo = environment.getContext().getZooReaderWriter(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java index 6220bb163c1..afcf127dce5 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.fate.FateTxId; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.TServerInstance; @@ -54,7 +55,7 @@ class CompactionDriver extends ManagerRepo { public static String createCompactionCancellationPath(InstanceId instanceId, TableId tableId) { - return Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId.canonical() + return ZooUtil.getRoot(instanceId) + Constants.ZTABLES + "/" + tableId.canonical() + Constants.ZTABLE_COMPACT_CANCEL_ID; } @@ -85,7 +86,8 @@ public long isReady(long tid, Manager manager) throws Exception { return 0; } - String zCancelID = createCompactionCancellationPath(manager.getInstanceID(), tableId); + String zCancelID = + createCompactionCancellationPath(manager.getContext().getInstanceID(), tableId); ZooReaderWriter zoo = manager.getContext().getZooReaderWriter(); if (Long.parseLong(new String(zoo.getData(zCancelID), UTF_8)) >= compactId) { @@ -96,7 +98,7 @@ public long isReady(long tid, Manager manager) throws Exception { } String deleteMarkerPath = - PreDeleteTable.createDeleteMarkerPath(manager.getInstanceID(), tableId); + PreDeleteTable.createDeleteMarkerPath(manager.getContext().getInstanceID(), tableId); if (zoo.exists(deleteMarkerPath)) { // table is being deleted throw new AcceptableThriftTableOperationException(tableId.canonical(), null, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/cancel/CancelCompactions.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/cancel/CancelCompactions.java index 6dad08986dc..0901edf9ead 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/cancel/CancelCompactions.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/cancel/CancelCompactions.java @@ -68,9 +68,9 @@ public void undo(long tid, Manager env) { public static void mutateZooKeeper(long tid, TableId tableId, Manager environment) throws Exception { - String zCompactID = Constants.ZROOT + "/" + environment.getInstanceID() + Constants.ZTABLES - + "/" + tableId + Constants.ZTABLE_COMPACT_ID; - String zCancelID = Constants.ZROOT + "/" + environment.getInstanceID() + Constants.ZTABLES + "/" + String zCompactID = environment.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + tableId + Constants.ZTABLE_COMPACT_ID; + String zCancelID = environment.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_CANCEL_ID; ZooReaderWriter zoo = environment.getContext().getZooReaderWriter(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java index 000d18ba674..77273d71d0d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.ManagerRepo; @@ -36,7 +37,7 @@ public class PreDeleteTable extends ManagerRepo { public static String createDeleteMarkerPath(InstanceId instanceId, TableId tableId) { - return Constants.ZROOT + "/" + instanceId + Constants.ZTABLES + "/" + tableId.canonical() + return ZooUtil.getRoot(instanceId) + Constants.ZTABLES + "/" + tableId.canonical() + Constants.ZTABLE_DELETE_MARKER; } @@ -58,7 +59,8 @@ public long isReady(long tid, Manager env) throws Exception { private void preventFutureCompactions(Manager environment) throws KeeperException, InterruptedException { - String deleteMarkerPath = createDeleteMarkerPath(environment.getInstanceID(), tableId); + String deleteMarkerPath = + createDeleteMarkerPath(environment.getContext().getInstanceID(), tableId); ZooReaderWriter zoo = environment.getContext().getZooReaderWriter(); zoo.putPersistentData(deleteMarkerPath, new byte[] {}, NodeExistsPolicy.SKIP); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java index b47223f1519..301f6e65e86 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java @@ -54,8 +54,8 @@ public Repo call(long tid, Manager manager) throws Exception { try { var context = manager.getContext(); NamespaceMapping.put(context.getZooReaderWriter(), - Constants.ZROOT + "/" + context.getInstanceID() + Constants.ZNAMESPACES, - namespaceInfo.namespaceId, namespaceInfo.namespaceName); + context.getZooKeeperRoot() + Constants.ZNAMESPACES, namespaceInfo.namespaceId, + namespaceInfo.namespaceName); TableManager.prepareNewNamespaceState(context, namespaceInfo.namespaceId, namespaceInfo.namespaceName, NodeExistsPolicy.OVERWRITE); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java index b1fcb9f30b9..ebcb2e73ce9 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java @@ -56,8 +56,8 @@ public Repo call(long id, Manager manager) throws Exception { Utils.getTableNameLock().lock(); try { - NamespaceMapping.rename(zoo, manager.getZooKeeperRoot() + Constants.ZNAMESPACES, namespaceId, - oldName, newName); + NamespaceMapping.rename(zoo, manager.getContext().getZooKeeperRoot() + Constants.ZNAMESPACES, + namespaceId, oldName, newName); manager.getContext().clearTableListCache(); } finally { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/rename/RenameTable.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/rename/RenameTable.java index 3fbbd5184b5..4d605f6df01 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/rename/RenameTable.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/rename/RenameTable.java @@ -83,8 +83,8 @@ public Repo call(long tid, Manager manager) throws Exception { final String newName = qualifiedNewTableName.getSecond(); final String oldName = qualifiedOldTableName.getSecond(); - final String tap = - manager.getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAME; + final String tap = manager.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + tableId + + Constants.ZTABLE_NAME; zoo.mutateExisting(tap, current -> { final String currentName = new String(current, UTF_8); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java index eb7ab83904f..4388ec65622 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java @@ -96,11 +96,10 @@ public Repo call(long tid, Manager manager) throws Exception { // suppress assignment of tablets to the server if (force) { ZooReaderWriter zoo = manager.getContext().getZooReaderWriter(); - var path = - ServiceLock.path(manager.getZooKeeperRoot() + Constants.ZTSERVERS + "/" + hostAndPort); + var zRoot = manager.getContext().getZooKeeperRoot(); + var path = ServiceLock.path(zRoot + Constants.ZTSERVERS + "/" + hostAndPort); ServiceLock.deleteLock(zoo, path); - path = ServiceLock - .path(manager.getZooKeeperRoot() + Constants.ZDEADTSERVERS + "/" + hostAndPort); + path = ServiceLock.path(zRoot + Constants.ZDEADTSERVERS + "/" + hostAndPort); zoo.putPersistentData(path.toString(), "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 60d80aaa448..ffdd6ba5066 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -124,7 +124,7 @@ public void upgradeZookeeper(@NonNull ServerContext context) { log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } - String zPath = Constants.ZROOT + "/" + context.getInstanceID() + Constants.ZNAMESPACES; + String zPath = context.getZooKeeperRoot() + Constants.ZNAMESPACES; byte[] namespacesData = zrw.getData(zPath); if (namespacesData.length != 0) { throw new IllegalStateException( diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java index 8840138c9ad..8cfd2d0c63b 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java @@ -19,6 +19,10 @@ package org.apache.accumulo.manager.tableOps.compact; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -35,7 +39,6 @@ import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.delete.PreDeleteTable; import org.apache.accumulo.server.ServerContext; -import org.easymock.EasyMock; import org.junit.jupiter.api.Test; public class CompactionDriverTest { @@ -51,17 +54,17 @@ public void testCancelId() throws Exception { final byte[] startRow = new byte[0]; final byte[] endRow = new byte[0]; - Manager manager = EasyMock.createNiceMock(Manager.class); - ServerContext ctx = EasyMock.createNiceMock(ServerContext.class); - ZooReaderWriter zrw = EasyMock.createNiceMock(ZooReaderWriter.class); - EasyMock.expect(manager.getInstanceID()).andReturn(instance).anyTimes(); - EasyMock.expect(manager.getContext()).andReturn(ctx); - EasyMock.expect(ctx.getZooReaderWriter()).andReturn(zrw); + Manager manager = createMock(Manager.class); + ServerContext ctx = createMock(ServerContext.class); + ZooReaderWriter zrw = createMock(ZooReaderWriter.class); + expect(ctx.getInstanceID()).andReturn(instance).anyTimes(); + expect(ctx.getZooReaderWriter()).andReturn(zrw).anyTimes(); + expect(manager.getContext()).andReturn(ctx).anyTimes(); final String zCancelID = CompactionDriver.createCompactionCancellationPath(instance, tableId); - EasyMock.expect(zrw.getData(zCancelID)).andReturn(Long.toString(cancelId).getBytes(UTF_8)); + expect(zrw.getData(zCancelID)).andReturn(Long.toString(cancelId).getBytes(UTF_8)); - EasyMock.replay(manager, ctx, zrw); + replay(manager, ctx, zrw); final CompactionDriver driver = new CompactionDriver(compactId, namespaceId, tableId, startRow, endRow); @@ -75,7 +78,7 @@ public void testCancelId() throws Exception { assertEquals(e.getType(), TableOperationExceptionType.OTHER); assertEquals(TableOperationsImpl.COMPACTION_CANCELED_MSG, e.getDescription()); - EasyMock.verify(manager, ctx, zrw); + verify(manager, ctx, zrw); } @Test @@ -89,20 +92,20 @@ public void testTableBeingDeleted() throws Exception { final byte[] startRow = new byte[0]; final byte[] endRow = new byte[0]; - Manager manager = EasyMock.createNiceMock(Manager.class); - ServerContext ctx = EasyMock.createNiceMock(ServerContext.class); - ZooReaderWriter zrw = EasyMock.createNiceMock(ZooReaderWriter.class); - EasyMock.expect(manager.getInstanceID()).andReturn(instance).anyTimes(); - EasyMock.expect(manager.getContext()).andReturn(ctx); - EasyMock.expect(ctx.getZooReaderWriter()).andReturn(zrw); + Manager manager = createMock(Manager.class); + ServerContext ctx = createMock(ServerContext.class); + ZooReaderWriter zrw = createMock(ZooReaderWriter.class); + expect(ctx.getInstanceID()).andReturn(instance).anyTimes(); + expect(ctx.getZooReaderWriter()).andReturn(zrw).anyTimes(); + expect(manager.getContext()).andReturn(ctx).anyTimes(); final String zCancelID = CompactionDriver.createCompactionCancellationPath(instance, tableId); - EasyMock.expect(zrw.getData(zCancelID)).andReturn(Long.toString(cancelId).getBytes(UTF_8)); + expect(zrw.getData(zCancelID)).andReturn(Long.toString(cancelId).getBytes(UTF_8)); String deleteMarkerPath = PreDeleteTable.createDeleteMarkerPath(instance, tableId); - EasyMock.expect(zrw.exists(deleteMarkerPath)).andReturn(true); + expect(zrw.exists(deleteMarkerPath)).andReturn(true); - EasyMock.replay(manager, ctx, zrw); + replay(manager, ctx, zrw); final CompactionDriver driver = new CompactionDriver(compactId, namespaceId, tableId, startRow, endRow); @@ -116,7 +119,7 @@ public void testTableBeingDeleted() throws Exception { assertEquals(e.getType(), TableOperationExceptionType.OTHER); assertEquals(TableOperationsImpl.TABLE_DELETED_MSG, e.getDescription()); - EasyMock.verify(manager, ctx, zrw); + verify(manager, ctx, zrw); } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index bb21b4efec4..d0c1bbf1cfc 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -353,49 +353,46 @@ public void upgradeZooKeeperTest() throws Exception { ServerContext context = createMock(ServerContext.class); ZooReaderWriter zrw = createStrictMock(ZooReaderWriter.class); + final var zkRoot = ZooUtil.getRoot(iid); expect(context.getInstanceID()).andReturn(iid).anyTimes(); expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes(); + expect(context.getZooKeeperRoot()).andReturn(zkRoot).anyTimes(); - zrw.recursiveDelete(Constants.ZROOT + "/" + iid.canonical() + "/tracers", - ZooUtil.NodeMissingPolicy.SKIP); + zrw.recursiveDelete(zkRoot + "/tracers", ZooUtil.NodeMissingPolicy.SKIP); expectLastCall().once(); Capture statCapture = newCapture(); - expect(zrw.getData(eq(Constants.ZROOT + "/" + iid.canonical() + "/root_tablet"), - capture(statCapture))).andAnswer(() -> { - Stat stat = statCapture.getValue(); - stat.setCtime(System.currentTimeMillis()); - stat.setMtime(System.currentTimeMillis()); - stat.setVersion(123); // default version - stat.setDataLength(zKRootV1.length); - statCapture.setValue(stat); - return zKRootV1; - }).once(); + expect(zrw.getData(eq(zkRoot + "/root_tablet"), capture(statCapture))).andAnswer(() -> { + Stat stat = statCapture.getValue(); + stat.setCtime(System.currentTimeMillis()); + stat.setMtime(System.currentTimeMillis()); + stat.setVersion(123); // default version + stat.setDataLength(zKRootV1.length); + statCapture.setValue(stat); + return zKRootV1; + }).once(); Capture byteCapture = newCapture(); - expect(zrw.overwritePersistentData(eq(Constants.ZROOT + "/" + iid.canonical() + "/root_tablet"), - capture(byteCapture), eq(123))).andReturn(true).once(); + expect(zrw.overwritePersistentData(eq(zkRoot + "/root_tablet"), capture(byteCapture), eq(123))) + .andReturn(true).once(); - expect(zrw.getData(eq(Constants.ZROOT + "/" + iid.canonical() + Constants.ZNAMESPACES))) - .andReturn(new byte[0]).once(); + expect(zrw.getData(eq(zkRoot + Constants.ZNAMESPACES))).andReturn(new byte[0]).once(); Map mockNamespaces = Map.of("ns1", "ns1name", "ns2", "ns2name"); - expect(zrw.getChildren(eq(Constants.ZROOT + "/" + iid.canonical() + Constants.ZNAMESPACES))) + expect(zrw.getChildren(eq(zkRoot + Constants.ZNAMESPACES))) .andReturn(List.copyOf(mockNamespaces.keySet())).once(); for (String ns : mockNamespaces.keySet()) { - Supplier pathMatcher = () -> eq(Constants.ZROOT + "/" + iid.canonical() - + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME); + Supplier pathMatcher = + () -> eq(zkRoot + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME); expect(zrw.getData(pathMatcher.get())).andReturn(mockNamespaces.get(ns).getBytes(UTF_8)) .once(); } byte[] mapping = NamespaceMapping.serialize(mockNamespaces); - expect( - zrw.putPersistentData(eq(Constants.ZROOT + "/" + iid.canonical() + Constants.ZNAMESPACES), - aryEq(mapping), eq(ZooUtil.NodeExistsPolicy.OVERWRITE))) - .andReturn(true).once(); + expect(zrw.putPersistentData(eq(zkRoot + Constants.ZNAMESPACES), aryEq(mapping), + eq(ZooUtil.NodeExistsPolicy.OVERWRITE))).andReturn(true).once(); for (String ns : mockNamespaces.keySet()) { - Supplier pathMatcher = () -> eq(Constants.ZROOT + "/" + iid.canonical() - + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME); + Supplier pathMatcher = + () -> eq(zkRoot + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME); zrw.delete(pathMatcher.get()); expectLastCall().once(); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 287bee16f32..53cba11d06f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -679,8 +679,8 @@ private MinorCompactionTask createMinorCompactionTask(long flushId, public long getFlushID() throws NoNodeException { try { - String zTablePath = Constants.ZROOT + "/" + tabletServer.getInstanceID() + Constants.ZTABLES - + "/" + extent.tableId() + Constants.ZTABLE_FLUSH_ID; + String zTablePath = tabletServer.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + extent.tableId() + Constants.ZTABLE_FLUSH_ID; String id = new String(context.getZooReaderWriter().getData(zTablePath), UTF_8); return Long.parseLong(id); } catch (InterruptedException | NumberFormatException e) { @@ -695,16 +695,16 @@ public long getFlushID() throws NoNodeException { } long getCompactionCancelID() { - String zTablePath = Constants.ZROOT + "/" + tabletServer.getInstanceID() + Constants.ZTABLES - + "/" + extent.tableId() + Constants.ZTABLE_COMPACT_CANCEL_ID; + String zTablePath = tabletServer.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + extent.tableId() + Constants.ZTABLE_COMPACT_CANCEL_ID; String id = new String(context.getZooCache().get(zTablePath), UTF_8); return Long.parseLong(id); } public Pair getCompactionID() throws NoNodeException { try { - String zTablePath = Constants.ZROOT + "/" + tabletServer.getInstanceID() + Constants.ZTABLES - + "/" + extent.tableId() + Constants.ZTABLE_COMPACT_ID; + String zTablePath = tabletServer.getContext().getZooKeeperRoot() + Constants.ZTABLES + "/" + + extent.tableId() + Constants.ZTABLE_COMPACT_ID; String[] tokens = new String(context.getZooReaderWriter().getData(zTablePath), UTF_8).split(","); diff --git a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java index 52827a4d69e..289f71d5049 100644 --- a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java @@ -31,7 +31,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; @@ -117,8 +116,7 @@ public void testExistingInstance() throws Exception { } ZooReaderWriter zrw = getCluster().getServerContext().getZooReaderWriter(); - final String zInstanceRoot = - Constants.ZROOT + "/" + client.instanceOperations().getInstanceId(); + final String zInstanceRoot = getCluster().getServerContext().getZooKeeperRoot(); while (!AccumuloStatus.isAccumuloOffline(zrw, zInstanceRoot)) { log.debug("Accumulo services still have their ZK locks held"); Thread.sleep(1000); diff --git a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java index 8e5b0765579..efb3bbbf314 100644 --- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java @@ -47,6 +47,7 @@ import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.ImportConfiguration; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -371,9 +372,11 @@ private boolean verifyMappingsFile(String destTableId) throws IOException { AccumuloCluster cluster = getCluster(); assertTrue(cluster instanceof MiniAccumuloClusterImpl); MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster; - String rootPath = mac.getConfig().getDir().getAbsolutePath(); FileSystem fs = getCluster().getFileSystem(); - FileStatus[] status = fs.listStatus(new Path(rootPath + "/accumulo/tables/" + destTableId)); + // the following path expects mini to be configured with a single volume + final Path tablePath = new Path(mac.getSiteConfiguration().get(Property.INSTANCE_VOLUMES) + "/" + + Constants.TABLE_DIR + "/" + destTableId); + FileStatus[] status = fs.listStatus(tablePath); for (FileStatus tabletDir : status) { var contents = fs.listStatus(tabletDir.getPath()); for (FileStatus file : contents) { diff --git a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java index 33588a882b6..41a3f76e639 100644 --- a/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ThriftServerBindsBeforeZooKeeperLockIT.java @@ -32,7 +32,6 @@ import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.util.MonitorUtil; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -135,13 +134,12 @@ public void testMonitorService() throws Exception { public void testManagerService() throws Exception { final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster(); try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - final InstanceId instanceID = client.instanceOperations().getInstanceId(); // Wait for the Manager to grab its lock while (true) { try { List locks = cluster.getServerContext().getZooReader() - .getChildren(Constants.ZROOT + "/" + instanceID + Constants.ZMANAGER_LOCK); + .getChildren(cluster.getServerContext().getZooKeeperRoot() + Constants.ZMANAGER_LOCK); if (!locks.isEmpty()) { break; } @@ -194,13 +192,12 @@ public void testManagerService() throws Exception { public void testGarbageCollectorPorts() throws Exception { final MiniAccumuloClusterImpl cluster = (MiniAccumuloClusterImpl) getCluster(); try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - InstanceId instanceID = client.instanceOperations().getInstanceId(); // Wait for the Manager to grab its lock while (true) { try { List locks = cluster.getServerContext().getZooReader() - .getChildren(Constants.ZROOT + "/" + instanceID + Constants.ZGC_LOCK); + .getChildren(cluster.getServerContext().getZooKeeperRoot() + Constants.ZGC_LOCK); if (!locks.isEmpty()) { break; } diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeManagerIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeManagerIT.java index 4a3002bd075..7e3e619936f 100644 --- a/test/src/main/java/org/apache/accumulo/test/VolumeManagerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/VolumeManagerIT.java @@ -24,6 +24,7 @@ import java.util.Map; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.NewTableConfiguration; @@ -50,6 +51,7 @@ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSit vol1 = config.getSiteConfig().get(Property.INSTANCE_VOLUMES.getKey()); assertTrue(vol1.contains("localhost")); vol2 = vol1.replace("localhost", "127.0.0.1"); + // order matters here, because this is parsed later to find the path to the table volumes config.setProperty(Property.INSTANCE_VOLUMES.getKey(), String.join(",", vol2, vol1)); // Set Volume specific HDFS overrides @@ -121,8 +123,11 @@ public void testHdfsConfigOverrides() throws Exception { // Confirm that table 1 has a block size of 10485760 FileSystem fs = this.cluster.getMiniDfs().getFileSystem(); - RemoteIterator iter1 = - fs.listFiles(new Path("/accumulo/tables/" + tid1), true); + // t1 is configured to use vol1, which is the second in the volumes list + final Path tablePath1 = + new Path(cluster.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).split(",")[1] + "/" + + Constants.TABLE_DIR + "/" + tid1); + RemoteIterator iter1 = fs.listFiles(tablePath1, true); while (iter1.hasNext()) { LocatedFileStatus stat = iter1.next(); if (stat.isFile()) { @@ -131,8 +136,11 @@ public void testHdfsConfigOverrides() throws Exception { } // Confirm that table 1 has a block size of 51200000 - RemoteIterator iter2 = - fs.listFiles(new Path("/accumulo/tables/" + tid2), true); + // t2 is configured to use vol2, which is the first in the volumes list + final Path tablePath2 = + new Path(cluster.getSiteConfiguration().get(Property.INSTANCE_VOLUMES).split(",")[0] + "/" + + Constants.TABLE_DIR + "/" + tid2); + RemoteIterator iter2 = fs.listFiles(tablePath2, true); while (iter2.hasNext()) { LocatedFileStatus stat = iter2.next(); if (stat.isFile()) { diff --git a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java index 540277d96db..3428349a2cc 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java @@ -46,6 +46,7 @@ import org.apache.accumulo.core.clientImpl.thrift.TableOperation; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.AgeOffStore; @@ -56,6 +57,7 @@ import org.apache.accumulo.core.fate.ZooStore; import org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.ManagerRepo; import org.apache.accumulo.manager.tableOps.TraceRepo; @@ -179,7 +181,8 @@ public Repo call(long tid, Manager environment) throws Exception { private static ZooKeeperTestingServer szk = null; private static ZooReaderWriter zk = null; - private static final String ZK_ROOT = "/accumulo/" + UUID.randomUUID(); + private static final InstanceId IID = InstanceId.of(UUID.randomUUID()); + private static final String ZK_ROOT = ZooUtil.getRoot(IID); private static final NamespaceId NS = NamespaceId.of("testNameSpace"); private static final TableId TID = TableId.of("testTable"); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooMutatorIT.java b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooMutatorIT.java index 639c52e02cd..bf860864665 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooMutatorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ZooMutatorIT.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.harness.WithTestNames; import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer; import org.junit.jupiter.api.Tag; @@ -84,7 +85,8 @@ public void concurrentMutatorTest() throws Exception { File newFolder = new File(tempDir, testName() + "/"); assertTrue(newFolder.isDirectory() || newFolder.mkdir(), "failed to create dir: " + newFolder); try (ZooKeeperTestingServer szk = new ZooKeeperTestingServer(newFolder)) { - szk.initPaths("/accumulo/" + InstanceId.of(UUID.randomUUID())); + final var iid = InstanceId.of(UUID.randomUUID()); + szk.initPaths(ZooUtil.getRoot(iid)); ZooReaderWriter zk = szk.getZooReaderWriter(); var executor = Executors.newFixedThreadPool(16); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java index fd9a908d868..5e4067f1bbd 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java @@ -46,7 +46,7 @@ public void test() throws Exception { Process backup = exec(Manager.class); try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { ZooReaderWriter writer = getCluster().getServerContext().getZooReaderWriter(); - String root = "/accumulo/" + client.instanceOperations().getInstanceId(); + String root = getCluster().getServerContext().getZooKeeperRoot(); // wait for 2 lock entries var path = ServiceLock.path(root + Constants.ZMANAGER_LOCK); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java index 06b8adc2c11..b8fc9814413 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java @@ -39,6 +39,7 @@ import java.util.regex.Pattern; import org.apache.accumulo.cluster.AccumuloCluster; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; @@ -226,7 +227,6 @@ public void testDeleteClone() throws Exception { AccumuloCluster cluster = getCluster(); assumeTrue(cluster instanceof MiniAccumuloClusterImpl); MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster; - String rootPath = mac.getConfig().getDir().getAbsolutePath(); // verify that deleting a new table removes the files c.tableOperations().create(table3); @@ -234,8 +234,12 @@ public void testDeleteClone() throws Exception { c.tableOperations().flush(table3, null, null, true); // check for files FileSystem fs = getCluster().getFileSystem(); - String id = c.tableOperations().tableIdMap().get(table3); - FileStatus[] status = fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)); + final String id = c.tableOperations().tableIdMap().get(table3); + + // the following path expects mini to be configured with a single volume + final Path tablePath = new Path(mac.getSiteConfiguration().get(Property.INSTANCE_VOLUMES) + + "/" + Constants.TABLE_DIR + "/" + id); + FileStatus[] status = fs.listStatus(tablePath); assertTrue(status.length > 0); // verify disk usage List diskUsage = c.tableOperations().getDiskUsage(Collections.singleton(table3)); @@ -244,7 +248,6 @@ public void testDeleteClone() throws Exception { // delete the table c.tableOperations().delete(table3); // verify its gone from the file system - Path tablePath = new Path(rootPath + "/accumulo/tables/" + id); if (fs.exists(tablePath)) { status = fs.listStatus(tablePath); assertTrue(status == null || status.length == 0); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java index bf8233125c9..1e6c36edefc 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java @@ -141,7 +141,9 @@ public void gcTest() throws Exception { TestIngest.ingest(c, cluster.getFileSystem(), params); log.info("Compacting the table {}", table); c.tableOperations().compact(table, null, null, true, true); - String pathString = cluster.getConfig().getDir() + "/accumulo/tables/1/*/*.rf"; + // the following path expects mini to be configured with a single volume + final String pathString = cluster.getSiteConfiguration().get(Property.INSTANCE_VOLUMES) + "/" + + Constants.TABLE_DIR + "/1/*/*.rf"; log.info("Counting files in path: {}", pathString); int before = countFiles(pathString); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java index 0f63b9f67d3..5bfce9b87d9 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TableIT.java @@ -28,10 +28,12 @@ import java.time.Duration; import org.apache.accumulo.cluster.AccumuloCluster; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.metadata.AccumuloTable; @@ -61,7 +63,6 @@ public void test() throws Exception { AccumuloCluster cluster = getCluster(); MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster; - String rootPath = mac.getConfig().getDir().getAbsolutePath(); try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { TableOperations to = c.tableOperations(); @@ -79,12 +80,15 @@ public void test() throws Exception { assertTrue(s.stream().findAny().isPresent()); FileSystem fs = getCluster().getFileSystem(); - assertTrue(fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length > 0); + // the following path expects mini to be configured with a single volume + final Path tablePath = new Path(mac.getSiteConfiguration().get(Property.INSTANCE_VOLUMES) + + "/" + Constants.TABLE_DIR + "/" + id); + assertTrue(fs.listStatus(tablePath).length > 0); to.delete(tableName); assertTrue(s.stream().findAny().isEmpty()); try { - assertEquals(0, fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length); + assertEquals(0, fs.listStatus(tablePath).length); } catch (FileNotFoundException ex) { // that's fine, too } diff --git a/test/src/main/java/org/apache/accumulo/test/lock/ServiceLockIT.java b/test/src/main/java/org/apache/accumulo/test/lock/ServiceLockIT.java index 3255ae701ee..abaa9698276 100644 --- a/test/src/main/java/org/apache/accumulo/test/lock/ServiceLockIT.java +++ b/test/src/main/java/org/apache/accumulo/test/lock/ServiceLockIT.java @@ -81,7 +81,8 @@ public class ServiceLockIT { @BeforeAll public static void setup() throws Exception { szk = new ZooKeeperTestingServer(tempDir); - szk.initPaths("/accumulo/" + InstanceId.of(UUID.randomUUID())); + final var iid = InstanceId.of(UUID.randomUUID()); + szk.initPaths(ZooUtil.getRoot(iid)); } @AfterAll