From b056c5facd3319ad39c416f5865a7a4ab468aa38 Mon Sep 17 00:00:00 2001 From: V_Galaxy Date: Mon, 13 May 2024 16:16:02 +0800 Subject: [PATCH] chore(pd): translate CJK comments to English (#2536) --- .../hugegraph/pd/client/AbstractClient.java | 1 - .../pd/client/AbstractClientStubProxy.java | 2 +- .../hugegraph/pd/client/ClientCache.java | 15 - .../hugegraph/pd/client/DiscoveryClient.java | 16 +- .../pd/client/DiscoveryClientImpl.java | 4 +- .../apache/hugegraph/pd/client/PDClient.java | 109 ++++--- .../apache/hugegraph/pd/client/PDConfig.java | 10 +- .../apache/hugegraph/pd/client/PDPulse.java | 3 +- .../hugegraph/pd/common/PartitionCache.java | 22 +- .../hugegraph/pd/common/PartitionUtils.java | 2 +- .../apache/hugegraph/pd/ConfigService.java | 7 +- .../org/apache/hugegraph/pd/IdService.java | 5 +- .../org/apache/hugegraph/pd/KvService.java | 1 - .../pd/PartitionInstructionListener.java | 2 +- .../apache/hugegraph/pd/PartitionService.java | 268 +++++++++--------- .../hugegraph/pd/PartitionStatusListener.java | 2 +- .../apache/hugegraph/pd/StoreNodeService.java | 159 ++++++----- .../hugegraph/pd/TaskScheduleService.java | 169 +++++------ .../apache/hugegraph/pd/config/PDConfig.java | 25 +- .../hugegraph/pd/meta/ConfigMetaStore.java | 2 +- .../apache/hugegraph/pd/meta/IdMetaStore.java | 32 ++- .../hugegraph/pd/meta/MetadataFactory.java | 2 +- .../pd/meta/MetadataRocksDBStore.java | 1 - .../hugegraph/pd/meta/MetadataStoreBase.java | 13 +- .../hugegraph/pd/meta/PartitionMeta.java | 34 +-- .../hugegraph/pd/meta/StoreInfoMeta.java | 17 +- .../hugegraph/pd/meta/TaskInfoMeta.java | 8 +- .../apache/hugegraph/pd/raft/KVOperation.java | 4 +- .../apache/hugegraph/pd/raft/RaftEngine.java | 22 +- .../hugegraph/pd/raft/RaftRpcClient.java | 2 +- .../hugegraph/pd/raft/RaftStateMachine.java | 1 - .../hugegraph/pd/raft/RaftTaskHandler.java | 2 +- .../hugegraph/pd/store/RaftKVStore.java | 4 +- .../src/assembly/static/conf/application.yml | 27 +- .../static/conf/application.yml.template | 21 +- hugegraph-pd/hg-pd-grpc/pom.xml | 8 - .../hg-pd-grpc/src/main/proto/metaTask.proto | 10 +- .../hg-pd-grpc/src/main/proto/metapb.proto | 124 ++++---- .../hg-pd-grpc/src/main/proto/pd_common.proto | 2 - .../hg-pd-grpc/src/main/proto/pd_pulse.proto | 30 +- .../hg-pd-grpc/src/main/proto/pd_watch.proto | 1 - .../hg-pd-grpc/src/main/proto/pdpb.proto | 123 ++++---- .../hugegraph/pd/boot/HugePDServer.java | 2 +- .../pd/pulse/PdInstructionSubject.java | 2 +- .../org/apache/hugegraph/pd/rest/API.java | 2 +- .../apache/hugegraph/pd/rest/GraphAPI.java | 12 +- .../apache/hugegraph/pd/rest/IndexAPI.java | 41 +-- .../apache/hugegraph/pd/rest/MemberAPI.java | 9 +- .../hugegraph/pd/rest/PartitionAPI.java | 47 +-- .../apache/hugegraph/pd/rest/ShardAPI.java | 2 - .../apache/hugegraph/pd/rest/StoreAPI.java | 48 ++-- .../pd/service/KvServiceGrpcImpl.java | 30 +- .../hugegraph/pd/service/PDRestService.java | 2 +- .../hugegraph/pd/service/PDService.java | 124 ++++---- .../hugegraph/pd/service/UpgradeService.java | 4 +- .../pd/upgrade/VersionUpgradeScript.java | 3 +- .../hugegraph/pd/watch/KvWatchSubject.java | 31 +- .../src/main/resources/application.yml | 29 +- .../test/resources/application-server0.yml | 10 - .../test/resources/application-server1.yml | 11 - .../test/resources/application-server2.yml | 10 - .../test/resources/application-server3.yml | 10 - .../hugegraph/pd/client/PDClientTest.java | 2 - .../pd/common/PartitionCacheTest.java | 1 - 64 files changed, 839 insertions(+), 905 deletions(-) diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java index 874ef6f67c..b83d7ba003 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java @@ -164,7 +164,6 @@ protected > RespT blockin log.error(method.getFullMethodName() + " exception, {}", e.getMessage()); if (e instanceof StatusRuntimeException) { if (retry < stubProxy.getHostCount()) { - // 网络不通，关掉之前连接，换host重新连接 synchronized (this) { stubProxy.setBlockingStub(null); } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java index 6ee3fcb625..a0bb181b75 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java @@ -42,7 +42,7 @@ public LinkedList getHostList() { public String nextHost() { String host = hostList.poll(); - hostList.offer(host); //移到尾部 + hostList.offer(host); return host; } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java index 868f8fae3a..0ebc285211 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java @@ -105,13 +105,6 @@ private KVPair getPair(int partId, GraphCache graph) { return null; } - /** - * 根据key的hashcode返回分区信息 - * - * @param graphName - * @param code - * @return - */ public KVPair getPartitionByCode(String graphName, long code) { try { GraphCache graph = initGraph(graphName); @@ -172,12 +165,6 @@ private void initCache() throws PDException { } } - /** - * 返回key所在的分区信息 - * - * @param key - * @return - */ public KVPair getPartitionByKey(String graphName, byte[] key) { int code = PartitionUtils.calcHashcode(key); return getPartitionByCode(graphName, code); @@ -193,8 +180,6 @@ public boolean update(String graphName, int partId, Partition partition) { RangeMap range = graph.getRange(); graph.addPartition(partId, partition); if (p != null) { - // old [1-3) 被 [2-3)覆盖了。当 [1-3) 变成[1-2) 不应该删除原先的[1-3) - // 当确认老的 start, end 都是自己的时候，才可以删除老的. (即还没覆盖） if (Objects.equals(partition.getId(), range.get(partition.getStartKey())) && Objects.equals(partition.getId(), range.get(partition.getEndKey() - 1))) { range.remove(range.getEntry(partition.getStartKey()).getKey()); diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java index c307b96216..d280b1344f 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java @@ -45,10 +45,10 @@ public abstract class DiscoveryClient implements Closeable, Discoverable { private final Timer timer = new Timer("serverHeartbeat", true); private final AtomicBoolean requireResetStub = new AtomicBoolean(false); - protected int period; //心跳周期 + protected int period; LinkedList pdAddresses = new LinkedList<>(); ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - private volatile int currentIndex; // 当前在用pd地址位置 + private volatile int currentIndex; private int maxTime = 6; private ManagedChannel channel = null; private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub registerStub; @@ -88,9 +88,6 @@ private R tryWithTimes(Function function, V v) { return null; } - /*** - * 按照pd列表重置stub - */ private void resetStub() { String errLog = null; for (int i = currentIndex + 1; i <= pdAddresses.size() + currentIndex; i++) { @@ -115,11 +112,6 @@ private void resetStub() { } } - /*** - * 按照某个pd的地址重置channel和stub - * @param singleAddress - * @throws PDException - */ private void resetChannel(String singleAddress) throws PDException { readWriteLock.writeLock().lock(); @@ -146,7 +138,7 @@ private void resetChannel(String singleAddress) throws PDException { } /*** - * 获取注册节点信息 + * Obtain the registration node information * @param query * @return */ @@ -167,7 +159,7 @@ public NodeInfos getNodeInfos(Query query) { } /*** - * 启动心跳任务 + * Start the heartbeat task */ @Override public void scheduleTask() { diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java index 049ca17a17..4f76d5ac9b 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java @@ -28,10 +28,10 @@ public class DiscoveryClientImpl extends DiscoveryClient { private final String id; - private final RegisterType type; // 心跳类型，备用 + private final RegisterType type; private final String version; private final String appName; - private final int times; // 心跳过期次数，备用 + private final int times; private final String address; private final Map labels; private final Consumer registerConsumer; diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java index 6c3eae4251..b38610c49f 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java @@ -53,7 +53,7 @@ import lombok.extern.slf4j.Slf4j; /** - * PD客户端实现类 + * PD client implementation class */ @Slf4j public class PDClient { @@ -78,7 +78,7 @@ private PDClient(PDConfig config) { } /** - * 创建PDClient对象，并初始化stub + * Create a PD client object and initialize the stub * * @param config * @return @@ -210,7 +210,6 @@ public void onError(Throwable throwable) { } private synchronized void closeStub(boolean closeWatcher) { - // TODO ManagedChannel 没有正常关闭 stubProxy.set(null); cache.reset(); @@ -308,7 +307,8 @@ private String getLeaderIp(PDGrpc.PDBlockingStub stub) { } /** - * Store注册，返回storeID，初次注册会返回新ID + * Store registration, the store ID will be returned, and the initial registration will + * return a new ID * * @param store * @return @@ -325,7 +325,7 @@ public long registerStore(Metapb.Store store) throws PDException { } /** - * 根据storeId返回Store对象 + * Returns the Store object based on the store ID * * @param storeId * @return @@ -348,7 +348,7 @@ public Metapb.Store getStore(long storeId) throws PDException { } /** - * 更新Store信息，包括上下线等 + * Update the store information, including online and offline * * @param store * @return @@ -368,7 +368,7 @@ public Metapb.Store updateStore(Metapb.Store store) throws PDException { } /** - * 返回活跃的Store + * Return to the active store * * @param graphName * @return @@ -400,7 +400,7 @@ public List getActiveStores() throws PDException { } /** - * 返回活跃的Store + * Return to the active store * * @param graphName * @return @@ -418,7 +418,7 @@ public List getAllStores(String graphName) throws PDException { } /** - * Store心跳，定期调用，保持在线状态 + * Store heartbeat, call regularly, stay online * * @param stats * @throws PDException @@ -452,7 +452,7 @@ private KVPair getKvPair(String graphName, byte[ } /** - * 查询Key所属分区信息 + * Query the partition to which the key belongs * * @param graphName * @param key @@ -461,7 +461,6 @@ private KVPair getKvPair(String graphName, byte[ */ public KVPair getPartition(String graphName, byte[] key) throws PDException { - // 先查cache，cache没有命中，在调用PD KVPair partShard = cache.getPartitionByKey(graphName, key); partShard = getKvPair(graphName, key, partShard); return partShard; @@ -477,7 +476,7 @@ public KVPair getPartition(String graphName, byt } /** - * 根据hashcode查询所属分区信息 + * Query the partition information based on the hashcode * * @param graphName * @param hashCode @@ -487,7 +486,6 @@ public KVPair getPartition(String graphName, byt public KVPair getPartitionByCode(String graphName, long hashCode) throws PDException { - // 先查cache，cache没有命中，在调用PD KVPair partShard = cache.getPartitionByCode(graphName, hashCode); if (partShard == null) { @@ -520,14 +518,14 @@ public KVPair getPartitionByCode(String graphNam } /** - * 获取Key的哈希值 + * Obtain the hash value of the key */ public int keyToCode(String graphName, byte[] key) { return PartitionUtils.calcHashcode(key); } /** - * 根据分区id返回分区信息, RPC请求 + * Returns partition information based on the partition ID and RPC request * * @param graphName * @param partId @@ -605,7 +603,7 @@ public void updateShardGroup(ShardGroup shardGroup) throws PDException { } /** - * 返回startKey和endKey跨越的所有分区信息 + * Returns information about all partitions spanned by the start and end keys * * @param graphName * @param startKey @@ -626,8 +624,7 @@ public List> scanPartitions(String graphN partitions.add(startPartShard); while (startPartShard.getKey().getEndKey() < endPartShard.getKey().getEndKey() - && startPartShard.getKey().getEndKey() < - PartitionUtils.MAX_VALUE /*排除最后一个分区*/) { + && startPartShard.getKey().getEndKey() < PartitionUtils.MAX_VALUE) { startPartShard = getPartitionByCode(graphName, startPartShard.getKey().getEndKey()); partitions.add(startPartShard); } @@ -635,7 +632,7 @@ public List> scanPartitions(String graphN } /** - * 根据条件查询分区信息 + * Query partition information based on conditions * * @return * @throws PDException @@ -654,12 +651,6 @@ public List getPartitionsByStore(long storeId) throws PDExcept return response.getPartitionsList(); } - /** - * 查找指定store上的指定partitionId - * - * @return - * @throws PDException - */ public List queryPartitions(long storeId, int partitionId) throws PDException { @@ -765,32 +756,30 @@ public Metapb.Partition delPartition(String graphName, int partitionId) throws P } /** - * 删除分区缓存 + * Delete the partitioned cache */ public void invalidPartitionCache(String graphName, int partitionId) { - // 检查是否存在缓存 if (null != cache.getPartitionById(graphName, partitionId)) { cache.removePartition(graphName, partitionId); } } /** - * 删除分区缓存 + * Delete the partitioned cache */ public void invalidPartitionCache() { - // 检查是否存在缓存 cache.removePartitions(); } /** - * 删除分区缓存 + * Delete the partitioned cache */ public void invalidStoreCache(long storeId) { cache.removeStore(storeId); } /** - * Hugegraph server 调用，Leader发生改变，更新缓存 + * Update the cache */ public void updatePartitionLeader(String graphName, int partId, long leaderStoreId) { KVPair partShard = null; @@ -817,7 +806,6 @@ public void updatePartitionLeader(String graphName, int partId, long leaderStore if (config.isEnableCache()) { if (shard == null) { - // 分区的shard中未找到leader，说明分区发生了迁移 cache.removePartition(graphName, partId); } } @@ -828,7 +816,7 @@ public void updatePartitionLeader(String graphName, int partId, long leaderStore } /** - * Hugegraph-store调用，更新缓存 + * Update the cache * * @param partition */ @@ -904,10 +892,7 @@ public Metapb.ClusterStats getClusterStats() throws PDException { } catch (Exception e) { log.error(method.getFullMethodName() + " exception, {}", e.getMessage()); if (e instanceof StatusRuntimeException) { - StatusRuntimeException se = (StatusRuntimeException) e; - //se.getStatus() == Status.UNAVAILABLE && if (retry < stubProxy.getHostCount()) { - // 网络不通，关掉之前连接，换host重新连接 closeStub(true); return blockingUnaryCall(method, req, ++retry); } @@ -938,7 +923,7 @@ public PDWatch getWatchClient() { } /** - * 返回Store状态信息 + * Returns the store status information */ public List getStoreStatus(boolean offlineExcluded) throws PDException { Pdpb.GetAllStoresRequest request = Pdpb.GetAllStoresRequest.newBuilder() @@ -1031,8 +1016,12 @@ public void changePeerList(String peerList) throws PDException { } /** - * 工作模式 - * Auto：自动分裂，每个Store上分区数达到最大值 + * Working mode + * Auto：If the number of partitions on each store reaches the maximum value, you need to + * specify the store group id. The store group id is 0, which is the default partition + * splitData(ClusterOp.OperationMode mode, int storeGroupId, List + * params) + * mode = Auto storeGroupId, params * * @throws PDException */ @@ -1046,9 +1035,11 @@ public void splitData() throws PDException { } /** - * 工作模式 - * Auto：自动分裂，每个Store上分区数达到最大值 - * Expert:专家模式，需要指定splitParams + * Working mode + * Auto：If the number of partitions on each store reaches the maximum value, you need to + * specify the store group id. The store group id is 0, which is the default partition + * Expert:Expert Mode，Specifier is required splitParams, limit SplitDataParam in the same + * store group * * @param mode * @param params @@ -1075,7 +1066,8 @@ public void splitGraphData(String graphName, int toCount) throws PDException { } /** - * 自动转移，达到每个Store上分区数量相同 + * To automatically transfer to the same number of partitions on each Store, it is + * recommended to use balancePartition(int storeGroupId) to specify the storeGroupId * * @throws PDException */ @@ -1090,12 +1082,13 @@ public void balancePartition() throws PDException { } /** - * //工作模式 - * // Auto：自动转移，达到每个Store上分区数量相同 - * // Expert:专家模式，需要指定transferParams + * Migrate partitions in manual mode + * //Working mode + * // Auto：Automatic transfer to the same number of partitions per Store + * // Expert:Expert Mode，Specifier is required transferParams * - * @param mode - * @param params + * @param params Designation transferParams, expert mode，request source store / target store + * in the same store group * @throws PDException */ public void movePartition(Pdpb.OperationMode mode, List params) throws @@ -1128,7 +1121,7 @@ public Metapb.PartitionStats getPartitionsStats(String graph, int partId) throws } /** - * 平衡不同store中leader的数量 + * Balance the number of leaders in different stores */ public void balanceLeaders() throws PDException { Pdpb.BalanceLeadersRequest request = Pdpb.BalanceLeadersRequest.newBuilder() @@ -1139,7 +1132,7 @@ public void balanceLeaders() throws PDException { } /** - * 从pd中删除store + * Remove the store from the PD */ public Metapb.Store delStore(long storeId) throws PDException { Pdpb.DetStoreRequest request = Pdpb.DetStoreRequest.newBuilder() @@ -1152,7 +1145,7 @@ public Metapb.Store delStore(long storeId) throws PDException { } /** - * 对rocksdb整体进行compaction + * Compaction on rocksdb as a whole * * @throws PDException */ @@ -1166,7 +1159,7 @@ public void dbCompaction() throws PDException { } /** - * 对rocksdb指定表进行compaction + * Compaction on rocksdb specified tables * * @param tableName * @throws PDException @@ -1182,9 +1175,9 @@ public void dbCompaction(String tableName) throws PDException { } /** - * 分区合并，把当前的分区缩容至toCount个 + * Merge partitions to reduce the current partition to toCount * - * @param toCount 缩容到分区的个数 + * @param toCount The number of partitions that can be scaled down * @throws PDException */ public void combineCluster(int toCount) throws PDException { @@ -1198,7 +1191,9 @@ public void combineCluster(int toCount) throws PDException { } /** - * 将单图缩容到 toCount个 + * Scaling a single image to toCount is similar to splitting to ensure that the number of + * partitions in the same store group is the same. + * If you have special requirements, you can consider migrating to other groups * * @param graphName graph name * @param toCount target count @@ -1228,7 +1223,7 @@ public void deleteShardGroup(int groupId) throws PDException { } /** - * 用于 store的 shard list重建 + * Used for the store's shard list rebuild * * @param groupId shard group id * @param shards shard list，delete when shards size is 0 @@ -1316,7 +1311,7 @@ public StubProxy(String[] hosts) { public String nextHost() { String host = hostList.poll(); - hostList.offer(host); //移到尾部 + hostList.offer(host); return host; } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java index a1c72a2bcf..822eda3d5a 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java @@ -19,11 +19,13 @@ public final class PDConfig { - //TODO multi-server + // TODO: multi-server private String serverHost = "localhost:9000"; - private long grpcTimeOut = 60000; // grpc调用超时时间 10秒 - // 是否接收PD异步通知 + // The timeout period for grpc call is 10 seconds + private long grpcTimeOut = 60000; + + // Whether to receive asynchronous PD notifications private boolean enablePDNotify = false; private boolean enableCache = false; @@ -59,8 +61,6 @@ public long getGrpcTimeOut() { @Deprecated public PDConfig setEnablePDNotify(boolean enablePDNotify) { this.enablePDNotify = enablePDNotify; - - // TODO 临时代码，hugegraph修改完后删除 this.enableCache = enablePDNotify; return this; } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java index 485417b917..08ad7b178a 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java @@ -78,7 +78,8 @@ public void onCompleted() { Notifier connectPartition(Listener listener); /** - * 切换成新的host。做 channel/host的检查，如果需要关闭，notifier调用close方法。 + * Switch to the new host. Do a channel/host check, and if you need to close, notifier calls + * the close method. * * @param host new host * @param notifier notifier diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java index 9bd233fd21..9b98cbbb03 100644 --- a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java @@ -35,18 +35,18 @@ import com.google.common.collect.TreeRangeMap; /** - * 放弃 copy on write 的方式 - * 1. 在 graph * partition 数量极多的时候，效率严重下降，不能用 + * abandon copy on write way + * 1. When the number of graph * partitions is extremely large, the efficiency is severely + * reduced and cannot be used */ public class PartitionCache { - // 读写锁对象 private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); private final Map locks = new HashMap<>(); Lock writeLock = readWriteLock.writeLock(); - // 每张图一个缓存 + // One cache per graph private volatile Map> keyToPartIdCache; - // graphName + PartitionID 组成 key + // graphName + PartitionID private volatile Map> partitionCache; private volatile Map shardGroupCache; private volatile Map storeCache; @@ -96,7 +96,7 @@ public void unlockGraph(String graphName) { } /** - * 根据 partitionId 返回分区信息 + * Returns partition information based on partitionId * * @param graphName * @param partId @@ -116,7 +116,7 @@ public KVPair getPartitionById(String graphName, } /** - * 返回 key 所在的分区信息 + * Returns the partition information where the key is located * * @param key * @return @@ -127,7 +127,7 @@ public KVPair getPartitionByKey(String graphName } /** - * 根据 key 的 hashcode 返回分区信息 + * Returns partition information based on the hashcode of the key * * @param graphName * @param code @@ -177,8 +177,10 @@ public boolean addPartition(String graphName, int partId, Metapb.Partition parti partitionCache.computeIfAbsent(graphName, k -> new HashMap<>()).put(partId, partition); if (old != null) { - // old [1-3) 被 [2-3) 覆盖了。当 [1-3) 变成 [1-2) 不应该删除原先的 [1-3) - // 当确认老的 start, end 都是自己的时候，才可以删除老的。(即还没覆盖） + // old [1-3] is covered by [2-3]. When [1-3) becomes [1-2], the original [1-3] + // should not be deleted + // When you confirm that the old start and end are your own, you can delete the + // old ones. (i.e. not covered yet) var graphRange = keyToPartIdCache.get(graphName); if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) && Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) { diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java index 0e35cc555e..1b24ec4c8f 100644 --- a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java @@ -22,7 +22,7 @@ public class PartitionUtils { public static final int MAX_VALUE = 0xffff; /** - * 计算key的hashcode + * compute key hashcode * * @param key * @return hashcode diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java index cc28c1b0a7..07ac73af43 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java @@ -71,7 +71,7 @@ public Metapb.GraphSpace setGraphSpace(Metapb.GraphSpace graphSpace) throws PDEx } /** - * 从存储中读取配置项，并覆盖全局的PDConfig对象 + * Read the configuration item from the storage and overwrite the global PD Config object * * @return */ @@ -120,8 +120,9 @@ public synchronized PDConfig setPartitionCount(int count) { } /** - * meta store中的数量 - * 由于可能会受分区分裂/合并的影响，原始的partition count不推荐使用 + * Meta store + * The original partition count is not recommended due to the fact that it may be affected by + * partition splitting/merging * * @return partition count of cluster * @throws PDException when io error diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java index 0c854d06db..a80052dacd 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java @@ -49,7 +49,8 @@ public void resetId(String key) throws PDException { } /** - * 获取自增循环不重复id, 达到上限后从0开始自增.自动跳过正在使用的cid + * Obtain the non-duplicate ID of the auto-increment cycle, and automatically increment from + * 0 after the upper limit is reached * * @param key * @param max @@ -65,7 +66,7 @@ public long getCId(String key, String name, long max) throws PDException { } /** - * 删除一个自增循环id + * Delete an auto-increment loop ID * * @param key * @param cid diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java index f31196f81c..c693a67b49 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java @@ -44,7 +44,6 @@ public class KvService { public static final char KV_DELIMITER = '@'; - // TODO 主前缀之后，增加类名做区分 private static final String TTL_PREFIX = "T"; private static final String KV_PREFIX = "K"; private static final String LOCK_PREFIX = "L"; diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionInstructionListener.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionInstructionListener.java index 2b1e4a6375..0ed4e855b4 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionInstructionListener.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionInstructionListener.java @@ -28,7 +28,7 @@ import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; /** - * 分区命令监听 + * Partition command listening */ public interface PartitionInstructionListener { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionService.java index c8ec3e3e7d..6ec779d27e 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionService.java @@ -52,7 +52,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 分区管理 + * Partition management */ @Slf4j public class PartitionService implements RaftStateListener { @@ -61,10 +61,10 @@ public class PartitionService implements RaftStateListener { private final StoreNodeService storeService; private final PartitionMeta partitionMeta; private final PDConfig pdConfig; - // 分区命令监听 + // Partition command listening private final List instructionListeners; - // 分区状态监听 + // Partition status listeners private final List statusListeners; public PartitionService(PDConfig config, StoreNodeService storeService) { @@ -83,7 +83,8 @@ public void init() throws PDException { public void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old, Metapb.StoreState status) { if (status == Metapb.StoreState.Tombstone) { - // Store被停机，通知所有该store所有分区，迁移数据 + // When the store is stopped, notify all partitions of the store and migrate + // the data storeOffline(store); } } @@ -103,7 +104,7 @@ public void onStoreRaftChanged(Metapb.Store store) { } /** - * 返回Key所属的partition + * return key partition * * @param graphName * @param key @@ -116,7 +117,7 @@ public Metapb.PartitionShard getPartitionShard(String graphName, byte[] key) thr } /** - * 根据hashcode返回所属的partition + * Returns the partition to which it belongs based on the hashcode * * @param graphName * @param code @@ -127,7 +128,7 @@ public Metapb.PartitionShard getPartitionByCode(String graphName, long code) thr if (code < 0 || code >= PartitionUtils.MAX_VALUE) { throw new PDException(Pdpb.ErrorType.NOT_FOUND_VALUE, "code error"); } - // 根据Code查找分区id，如果没有找到，创建新的分区 + // Find the partition ID based on the code, and if it doesn't find, create a new partition Metapb.Partition partition = partitionMeta.getPartitionByCode(graphName, code); if (partition == null) { @@ -152,7 +153,7 @@ public Metapb.PartitionShard getPartitionByCode(String graphName, long code) thr } /** - * 根据ID返回分区信息 + * Returns partition information based on ID * * @param graphName * @param partId @@ -168,7 +169,6 @@ public Metapb.PartitionShard getPartitionShardById(String graphName, int partId) Metapb.PartitionShard partShard = Metapb.PartitionShard.newBuilder() .setPartition(partition) - // 此处需要返回正确的leader，暂时默认取第一个 .setLeader(storeService.getLeader( partition, 0)) .build(); @@ -185,7 +185,7 @@ public List getPartitionById(int partId) throws PDException { } /** - * 获取图的所有分区 + * Get all partitions of the graph */ public List getPartitions() { return partitionMeta.getPartitions(); @@ -199,7 +199,7 @@ public List getPartitions(String graphName) { } /** - * 查找在store上的所有分区 + * Find all the partitions on the store * * @param store * @return @@ -223,7 +223,7 @@ public List getPartitionByStore(Metapb.Store store) throws PDE } /** - * 产生一个新的分区 + * Creates a new partition * * @param graphName * @return @@ -232,7 +232,7 @@ private Metapb.Partition newPartition(String graphName, long code) throws PDExce Metapb.Graph graph = partitionMeta.getAndCreateGraph(graphName); int partitionSize = PartitionUtils.MAX_VALUE / graph.getPartitionCount(); if (PartitionUtils.MAX_VALUE % graph.getPartitionCount() != 0) { - // 有余数，分区除不尽 + // There is a remainder, and the partition is inexhaustible partitionSize++; } @@ -240,12 +240,12 @@ private Metapb.Partition newPartition(String graphName, long code) throws PDExce long startKey = (long) partitionSize * partitionId; long endKey = (long) partitionSize * (partitionId + 1); - // 检查本地 + // Check Local Metapb.Partition partition = partitionMeta.getPartitionById(graphName, partitionId); if (partition == null) { storeService.allocShards(null, partitionId); - // 分配store + // Assign a store partition = Metapb.Partition.newBuilder() .setId(partitionId) .setVersion(0) @@ -264,11 +264,12 @@ private Metapb.Partition newPartition(String graphName, long code) throws PDExce } /** - * 计算Key所属的分区，此处采用Hash映射的方法。 + * compute graph partition id。partition gap * store group id + offset * - * @param graphName - * @param key - * @return + * @param graph graph + * @param offset offset + * @return new partition id + * @throws PDException */ protected int getPartitionId(String graphName, byte[] key) throws PDException { int code = PartitionUtils.calcHashcode(key); @@ -277,8 +278,9 @@ protected int getPartitionId(String graphName, byte[] key) throws PDException { } /** - * 获取key范围所跨越的所有分区 - * 暂时使用hashcode计算，正常做法，基于key进行查询 + * Gets all partitions spanned by the key range + * For the time being, hashcode is used for calculation, and the normal practice is to query + * based on the key * * @param graphName * @param startKey @@ -296,7 +298,8 @@ public List scanPartitions(String graphName, byte[] start partShards.add( Metapb.PartitionShard.newBuilder() .setPartition(partition) - // 此处需要返回正确的leader，暂时默认取第一个 + // Here you need to return the correct leader, and + // temporarily default to the first one .setLeader(storeService.getLeader(partition, 0)) .build() ); @@ -314,7 +317,7 @@ public synchronized long updatePartition(List partitions) thro } /** - * 更新分区以及图的状态 + * Update the status of partitions and graphs * * @param graph * @param partId @@ -350,7 +353,6 @@ public synchronized long removePartition(String graphName, int partId) throws PD partitionMeta.reload(); onPartitionRemoved(partition); - // source中有些是 offline的，删除后，需要更新图的状态 try { Metapb.PartitionState state = Metapb.PartitionState.PState_Normal; for (Metapb.Partition pt : partitionMeta.getPartitions(partition.getGraphName())) { @@ -381,7 +383,7 @@ public Metapb.PartitionStats getPartitionStats(String graphName, int partitionId } /** - * 获取图的分区状态 + * Get the partition status of the graph */ public List getPartitionStatus(String graphName) throws PDException { @@ -389,7 +391,7 @@ public List getPartitionStatus(String graphName) } /** - * 返回图的信息 + * Returns the information of the graph */ public List getGraphs() throws PDException { return partitionMeta.getGraphs(); @@ -400,7 +402,7 @@ public Metapb.Graph getGraph(String graphName) throws PDException { } /** - * 删除图以及图的所有分区 + * Delete the diagram and all partitions of the diagram */ public Metapb.Graph delGraph(String graphName) throws PDException { log.info("delGraph {}", graphName); @@ -414,7 +416,7 @@ public Metapb.Graph delGraph(String graphName) throws PDException { } /** - * 修改图信息，需要通知到store + * To modify the graph information, you need to notify the store */ public synchronized Metapb.Graph updateGraph(Metapb.Graph graph) throws PDException { Metapb.Graph lastGraph = partitionMeta.getAndCreateGraph(graph.getGraphName()); @@ -439,12 +441,11 @@ public synchronized Metapb.Graph updateGraph(Metapb.Graph graph) throws PDExcept .build(); partitionMeta.updateGraph(graph); - // 分区数发生改变 + // The number of partitions has changed if (lastGraph.getPartitionCount() != graph.getPartitionCount()) { log.info("updateGraph graph: {}, partition count changed from {} to {}", graph.getGraphName(), lastGraph.getPartitionCount(), graph.getPartitionCount()); - // TODO 修改图的分区数，需要进行数据迁移。 } return graph; } @@ -468,7 +469,7 @@ public Map> getCommittedIndexStats() throws PDException } /** - * 存储被下线，迁移分区数据 + * The storage is taken offline and the partition data is migrated * * @param store */ @@ -491,7 +492,7 @@ public void storeOffline(Metapb.Store store) { } /** - * 存储被下线，迁移分区数据 + * The storage is taken offline and the partition data is migrated */ public synchronized void shardOffline(Metapb.Partition partition, long storeId) { try { @@ -537,7 +538,7 @@ private boolean isShardListEquals(List list1, List l } /** - * 重新分配shard + * Reassign shards * * @param graph * @param partition @@ -571,7 +572,7 @@ public synchronized void reallocPartitionShards(String graphName, int partitionI } /** - * 迁移分区副本 + * Migrate partition copies */ public synchronized void movePartitionsShard(Integer partitionId, long fromStore, long toStore) { @@ -599,7 +600,7 @@ public synchronized void movePartitionsShard(Integer partitionId, long fromStore // storeService.updateShardGroup(partitionId, shards, -1, -1); // storeService.onShardGroupStatusChanged(shardGroup, newShardGroup); fireChangeShard(partition, shards, ConfChangeType.CONF_CHANGE_TYPE_ADJUST); - // shard group和 graph无关，迁移一个就够了 + // Shard groups have nothing to do with Graph, just one is enough break; } } catch (PDException e) { @@ -608,9 +609,9 @@ public synchronized void movePartitionsShard(Integer partitionId, long fromStore } /** - * 把集群中所有的分区，拆成split + * Split all partitions in the cluster into splits * - * @param splits 拆分分区 + * @param splits Split partitions */ public synchronized void splitPartition(List> splits) throws PDException { @@ -631,13 +632,12 @@ public synchronized void splitPartition(List> splits) t } /** - * 分区分裂， 把一个图拆分到N 个 + * Partition splitting, splitting a graph into N pieces * * @param graph graph * @param toCount target count * @throws PDException */ - public synchronized void splitPartition(Metapb.Graph graph, int toCount) throws PDException { var partitionCount = getPartitions(graph.getGraphName()).size(); @@ -656,7 +656,7 @@ public synchronized void splitPartition(Metapb.Graph graph, int toCount) throws " current partition count"); } - // 由于是整数倍数，扩充因子为 toCount / current count + // Since it is an integer multiple,The enrichment factor is toCount / current count var splitCount = toCount / partitionCount; var list = new ArrayList>(); for (int i = 0; i < partitionCount; i++) { @@ -677,7 +677,7 @@ private synchronized void splitPartition(Metapb.Graph graph, splits.sort(Comparator.comparing(KVPair::getKey)); log.info("split partition, graph: {}, splits:{}", graph, splits); - // 从最后一个partition下标开始 + // Start with the last partition subscript var i = getPartitions(graph.getGraphName()).size(); for (var pair : splits) { @@ -688,7 +688,7 @@ private synchronized void splitPartition(Metapb.Graph graph, long splitLen = (partition.getEndKey() - partition.getStartKey()) / splitCount; List newPartitions = new ArrayList<>(); - // 第一个分区也就是原分区 + // The first partition is the original partition newPartitions.add(partition.toBuilder() .setStartKey(partition.getStartKey()) .setEndKey(partition.getStartKey() + splitLen) @@ -724,8 +724,10 @@ private synchronized void splitPartition(Metapb.Graph graph, if (j != 0) { partitionMeta.updatePartition(newPartition); } - // 创建shard group，如果为空，则按照partition的shard group为蓝本，去创建，保证在一个机器上 - // 如果存在，则由于各个图的分区数量不一样，需要store端复制到其他机器上 + // Create a shard group, if it is empty, create it according to the shard + // group of the partition, and ensure that it is on one machine + // If it exists, the number of partitions in each graph is not the same, and + // the store side needs to be copied to other machines var shardGroup = storeService.getShardGroup(newPartition.getId()); if (shardGroup == null) { shardGroup = storeService.getShardGroup(partition.getId()).toBuilder() @@ -735,7 +737,7 @@ private synchronized void splitPartition(Metapb.Graph graph, updateShardGroupCache(shardGroup); } - // 做shard list的检查 + // check shard list if (shardGroup.getShardsCount() != pdConfig.getPartition().getShardCount()) { storeService.reallocShards(shardGroup); } @@ -746,11 +748,12 @@ private synchronized void splitPartition(Metapb.Graph graph, .build(); fireSplitPartition(partition, splitPartition); - // 修改Partition状态为下线，任务完成后恢复为上线 + // Change the partition status to Offline, and resume the partition status to + // Offline after the task is completed updatePartitionState(partition.getGraphName(), partition.getId(), Metapb.PartitionState.PState_Offline); - // 记录事务 + // Record transactions var task = MetaTask.Task.newBuilder().setPartition(partition) .setSplitPartition(splitPartition) .build(); @@ -761,8 +764,8 @@ private synchronized void splitPartition(Metapb.Graph graph, } /** - * 转移leader到其他shard上. - * 转移一个partition即可 + * transfer leader to other shard 。 + * Just transfer a partition */ public void transferLeader(Integer partId, Metapb.Shard shard) { try { @@ -784,16 +787,17 @@ public void transferLeader(Integer partId, Metapb.Shard shard) { } /** - * 分区合并，将整个集群的分区数，合并到toCount个 + * // todo : Check the corresponding store group and check the logic + * Partition merging: Merges the number of partitions in the entire cluster into toCount * - * @param toCount 目标分区数 + * @param toCount The number of partitions to be targeted * @throws PDException when query errors */ public void combinePartition(int toCount) throws PDException { int shardsTotalCount = getShardGroupCount(); for (var graph : getGraphs()) { - // 对所有大于toCount分区的图，都进行缩容 + // All graphs larger than the toCount partition are scaled in if (graph.getPartitionCount() > toCount) { combineGraphPartition(graph, toCount, shardsTotalCount); } @@ -801,19 +805,18 @@ public void combinePartition(int toCount) throws PDException { } /** - * 针对单个图，进行分区合并 + * For a single graph, perform partition merging * * @param graphName the name of the graph * @param toCount the target partition count * @throws PDException when query errors */ - public void combineGraphPartition(String graphName, int toCount) throws PDException { combineGraphPartition(getGraph(graphName), toCount, getShardGroupCount()); } /** - * 单图合并的内部实现 + * Internal implementation of single-graph merging * * @param graph the name of the graph * @param toCount the target partition count @@ -845,22 +848,22 @@ private synchronized void combineGraphPartition(Metapb.Graph graph, int toCount, throw new PDException(3, "Graph Combine process exists"); } - // 按照 key start 排序，合并后的key range 是连续的 + // According to key start sort var partitions = getPartitions(graph.getGraphName()).stream() .sorted(Comparator.comparing( Metapb.Partition::getStartKey)) .collect(Collectors.toList()); - // 分区编号不一定是连续的 + // Partition numbers do not have to be sequential var sortPartitions = getPartitions(graph.getGraphName()) .stream() .sorted(Comparator.comparing(Metapb.Partition::getId)) .collect(Collectors.toList()); var groupSize = partitions.size() / toCount; // merge group size - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 共12个分区, 合并成4个 - // 方案：0,1,2 => 0, 3,4,5 -> 1, 6,7,8 ->2, 9,10,11 -> 3 - // 保证分区的连续性. + // 0~12 to 4 partitions + // scheme：0,1,2 => 0, 3,4,5 -> 1, 6,7,8 ->2, 9,10,11 -> 3 + // Ensure the continuity of partitions for (int i = 0; i < toCount; i++) { var startKey = partitions.get(i * groupSize).getStartKey(); var endKey = partitions.get(i * groupSize + groupSize - 1).getEndKey(); @@ -874,7 +877,7 @@ private synchronized void combineGraphPartition(Metapb.Graph graph, int toCount, for (int j = 0; j < groupSize; j++) { var partition = partitions.get(i * groupSize + j); - // 分区id相同，就跳过 + // If the partition ID is the same, skip it if (i == partition.getId()) { continue; } @@ -888,12 +891,12 @@ private synchronized void combineGraphPartition(Metapb.Graph graph, int toCount, .setKeyEnd(partition.getEndKey()) .build(); taskInfoMeta.addMovePartitionTask(partition, movePartition); - // source 下线 + // source is offline updatePartitionState(partition.getGraphName(), partition.getId(), Metapb.PartitionState.PState_Offline); fireMovePartition(partition, movePartition); } - // target 下线 + // target offline updatePartitionState(targetPartition.getGraphName(), targetPartition.getId(), Metapb.PartitionState.PState_Offline); } @@ -902,7 +905,7 @@ private synchronized void combineGraphPartition(Metapb.Graph graph, int toCount, } /** - * 通过 storeService 获取 raft group 总数 + * get raft group count from storeService * * @return the count of raft groups */ @@ -917,30 +920,31 @@ private int getShardGroupCount() { } /** - * 判断图分区是否能够从from合并到to个 + * Determine whether the graph partition can be retrieved from f to t * - * @param fromCount 现在的分区数 - * @param toCount 目标分区数 + * @param fromCount The number of partitions now + * @param toCount The number of partitions to be targeted * @return true when available , or otherwise */ private boolean checkTargetCount(int fromCount, int toCount, int shardCount) { - // 要介于 1 ~ N 中间，而且可以整除 + // It should be between 1 ~ N and divisible return toCount >= 1 && toCount < fromCount && fromCount % toCount == 0 && toCount < shardCount; } /** - * 处理分区心跳， 记录Leader信息 - * 检查term和version，比较是否是最新的消息 + * Process partition heartbeats and record leader information + * Check the term and version to see if it's the latest message * * @param stats */ public void partitionHeartbeat(Metapb.PartitionStats stats) throws PDException { Metapb.ShardGroup shardGroup = storeService.getShardGroup(stats.getId()); - // shard group version changes - // (shard group 由pd控制, 在分裂等操作后，可能出现短暂不一致的情况，以pd为准） - // store控制shard leader + // shard group version changes or leader changes + // (The shard group is controlled by the PD, and there may be brief inconsistencies after + // operations such as splitting, subject to PD) + // store Upload the final one raft group data if (shardGroup != null && (shardGroup.getVersion() < stats.getLeaderTerm() || shardGroup.getConfVer() < stats.getConfVer())) { @@ -954,13 +958,13 @@ public void partitionHeartbeat(Metapb.PartitionStats stats) throws PDException { // partitionMeta.getAndCreateGraph(partition.getGraphName()); checkShardState(partition, stats); } - // 统计信息 + // statistics partitionMeta.updatePartitionStats(stats.toBuilder() .setTimestamp(System.currentTimeMillis()).build()); } /** - * 检查shard状态，离线shard影响到分区状态 + * Check the shard status, offline shard affects the partition status * * @param stats */ @@ -1000,7 +1004,7 @@ public void addStatusListener(PartitionStatusListener listener) { } /** - * 发起改变shard命令 + * Initiates the Change Shard command * * @param changeType */ @@ -1028,7 +1032,7 @@ public void changeShard(int groupId, List shards) throws PDExcepti } /** - * 发送分区分裂消息 + * Send a partition split message * * @param partition */ @@ -1045,7 +1049,7 @@ protected void fireSplitPartition(Metapb.Partition partition, SplitPartition spl } /** - * 发送Leader切换消息 + * Send a Leader Switchover message */ protected void fireTransferLeader(Metapb.Partition partition, TransferLeader transferLeader) { log.info("fireTransferLeader partition: {}-{}， leader :{}", @@ -1060,10 +1064,10 @@ protected void fireTransferLeader(Metapb.Partition partition, TransferLeader tra } /** - * 发送分区移动数据的消息 + * Send a message to the partition to move data * - * @param partition 原分区 - * @param movePartition 目标分区，包含 key range + * @param partition Original partition + * @param movePartition Target partition，contains key range */ protected void fireMovePartition(Metapb.Partition partition, MovePartition movePartition) { log.info("fireMovePartition partition: {} -> {}", @@ -1106,7 +1110,7 @@ protected void fireChangePartitionKeyRange(Metapb.Partition partition, } /** - * 处理图迁移任务 + * Handle graph migration tasks * * @param task */ @@ -1125,7 +1129,7 @@ public synchronized void handleMoveTask(MetaTask.Task task) throws PDException { task.getPartition().getId(), task.getMovePartition().getTargetPartition().getId(), task.getState()); - // 已经被处理（前面有failed) + // HAS BEEN PROCESSED(There is it in front) if (pdMetaTask != null) { var newTask = pdMetaTask.toBuilder().setState(task.getState()).build(); taskInfoMeta.updateMovePartitionTask(newTask); @@ -1153,10 +1157,10 @@ public synchronized void handleMoveTask(MetaTask.Task task) throws PDException { } /** - * 当所有的迁移子任务成功: - * 1. 发送清理source分区指令 - * 2. 设置target上线, 更新key range, 更新 graph partition count - * 3. 删除move task，任务结束 + * When all migration subtasks succeed: + * 1. Send cleanup source partition directives + * 2. Set up target online，renewal key range, renewal graph partition count + * 3. delete move task，mission ended * * @param subTasks all move sub tasks * @param graphName graph name @@ -1175,20 +1179,21 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph for (MetaTask.Task subTask : subTasks) { var source = subTask.getPartition(); var targetPartition = subTask.getMovePartition().getTargetPartition(); - // 是否处理过 + // Whether it has been dealt with or not if (!targetPartitionIds.contains(targetPartition.getId())) { - // 更新range + // renewal range var old = getPartitionById(targetPartition.getGraphName(), targetPartition.getId()); var newPartition = Metapb.Partition.newBuilder(old) .setStartKey(targetPartition.getStartKey()) .setEndKey(targetPartition.getEndKey()) .setState(Metapb.PartitionState.PState_Normal) .build(); - // 在 key range之前更新，避免store没有分区的问题, 需要到pd查询 + // Update before the key range to avoid the problem that the store does not have + // a partition and needs to be queried to the pd updatePartition(List.of(newPartition)); targetPartitions.add(newPartition); - // 发送key range 变更消息 + // Send key range change messages PartitionKeyRange partitionKeyRange = PartitionKeyRange.newBuilder() .setPartitionId(old.getId()) .setKeyStart( @@ -1196,12 +1201,13 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph .setKeyEnd( targetPartition.getEndKey()) .build(); - // 通知store + // Notice store fireChangePartitionKeyRange( old.toBuilder().setState(Metapb.PartitionState.PState_Normal).build(), partitionKeyRange); - // 将 target 设置为上线. source 理论上可能被删掉，所以不处理 + // Set Target to go live. source could theoretically be deleted, so it is not + // processed updatePartitionState(newPartition.getGraphName(), newPartition.getId(), Metapb.PartitionState.PState_Normal); @@ -1213,7 +1219,9 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph .setKeyEnd(source.getEndKey()) .setCleanType( CleanType.CLEAN_TYPE_EXCLUDE_RANGE) - // target 的 partition只需要清理数据，不需要删除分区 + // The partition of the target only + // needs to clean up the data, and does + // not need to delete the partition .setDeletePartition(!deleteFlags.contains( source.getId())) .build(); @@ -1226,14 +1234,14 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph CleanType.CLEAN_TYPE_EXCLUDE_RANGE, cleanPartition.getDeletePartition()); - // 清理掉被移动分区的数据 + // Clean up the data of the partition to be moved fireCleanPartition(source, cleanPartition); } - // 更新key range, 本地更新，client更新 + // renewal key range, Local updates，client renewal // updatePartition(targetPartitions); - // 更新target 分区状态，source 可能被删掉，所以不处理 + // renewal target Partition status, source may be deleted, so do not process targetPartitions.forEach(p -> { try { updatePartitionState(p.getGraphName(), p.getId(), @@ -1245,21 +1253,21 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph partitionMeta.reload(); - // 更新graph partition count + // renewal graph partition count var graph = getGraph(graphName).toBuilder() .setPartitionCount(targetPartitionIds.size()) .build(); updateGraph(graph); - // 事务完成 + // The transaction is complete taskInfoMeta.removeMoveTaskPrefix(graphName); } /** - * 如果缩容任务有失败的，回滚合并操作 - * 1. 清理原来的target 分区，将迁移过来的数据再删掉 - * 2. 将source/target 分区设置为上线 - * 3. 删除task，任务结束 + * If the scale-in task fails, roll back the merge operation + * 1. Clean up the original target partition and delete the migrated data + * 2. Set the source/target partition to go live + * 3. Delete the task, and the task ends * * @param graphName graph name * @param taskInfoMeta task info meta @@ -1267,12 +1275,12 @@ private void handleMoveTaskAllSuccess(List subTasks, String graph */ private void handleMoveTaskIfFailed(String graphName, TaskInfoMeta taskInfoMeta) throws PDException { - // 发送清理target分区的任务, 回滚target分区 + // Send cleanup target partition tasks,rollback target partition var targetPartitionIds = new HashSet(); for (var metaTask : taskInfoMeta.scanMoveTask(graphName)) { var source = metaTask.getPartition(); - // 设置 source 为上线 + // Set source to upline updatePartitionState(source.getGraphName(), source.getId(), Metapb.PartitionState.PState_Normal); var movedPartition = metaTask.getMovePartition().getTargetPartition(); @@ -1294,16 +1302,16 @@ private void handleMoveTaskIfFailed(String graphName, TaskInfoMeta taskInfoMeta) fireCleanPartition(targetPartition, cleanPartition); targetPartitionIds.add(targetPartition.getId()); - // 设置target 上线 + // Set Target online updatePartitionState(targetPartition.getGraphName(), targetPartition.getId(), Metapb.PartitionState.PState_Normal); } - // 清理掉任务列表 + // Clean up the task list taskInfoMeta.removeMoveTaskPrefix(graphName); } /** - * 处理clean task + * dispose clean task * * @param task clean task */ @@ -1316,7 +1324,7 @@ public void handleCleanPartitionTask(MetaTask.Task task) { task.getState() ); - // 如果失败重试？ + // If it fails, try again? } public synchronized void handleSplitTask(MetaTask.Task task) throws PDException { @@ -1366,7 +1374,7 @@ private void handleSplitTaskAllSuccess(List subTasks, String grap var source = subTask.getPartition(); var newPartition = subTask.getSplitPartition().getNewPartitionList().get(0); - // 发送key range 变更消息 + // Send key range change messages PartitionKeyRange partitionKeyRange = PartitionKeyRange.newBuilder() .setPartitionId(source.getId()) .setKeyStart( @@ -1374,16 +1382,18 @@ private void handleSplitTaskAllSuccess(List subTasks, String grap .setKeyEnd( newPartition.getEndKey()) .build(); - // 通知store + // Notice store fireChangePartitionKeyRange(source, partitionKeyRange); - // 将 target 设置为上线. source 理论上可能被删掉，所以不处理 + // Set Target to go live. source could theoretically be deleted, so it is not processed CleanPartition cleanPartition = CleanPartition.newBuilder() .setKeyStart(newPartition.getStartKey()) .setKeyEnd(newPartition.getEndKey()) .setCleanType( CleanType.CLEAN_TYPE_KEEP_RANGE) - // target 的 partition只需要清理数据，不需要删除分区 + // The partition of the target only + // needs to clean up the data, and does + // not need to delete the partition .setDeletePartition(false) .build(); @@ -1397,7 +1407,7 @@ private void handleSplitTaskAllSuccess(List subTasks, String grap fireCleanPartition(source, cleanPartition); - // 更新partition state + // renewal partition state for (var sp : subTask.getSplitPartition().getNewPartitionList()) { partitions.add( sp.toBuilder().setState(Metapb.PartitionState.PState_Normal).build()); @@ -1419,13 +1429,13 @@ private void handleSplitTaskAllSuccess(List subTasks, String grap storeService.getShardGroups().size()); } - // 更新graph partition count + // renewal graph partition count var newGraph = graph.toBuilder() .setPartitionCount(graph.getPartitionCount() + addedPartitions) .build(); updateGraph(newGraph); - // 事务完成 + // The transaction is complete taskInfoMeta.removeSplitTaskPrefix(graphName); } @@ -1452,18 +1462,18 @@ private void handleSplitTaskIfFailed(List subTasks, String graphN updatePartitionState(partition.getGraphName(), partition.getId(), Metapb.PartitionState.PState_Normal); } - // 清理掉任务列表 + // Clean up the task list taskInfoMeta.removeSplitTaskPrefix(graphName); } /** - * 接收到Leader改变的消息 - * 更新图状态，触发分区变更 + * todo : What is the impact of partition changes?? + * Received a message that the leader has changed + * Update the status of the graph and trigger a partition change */ protected void onPartitionChanged(Metapb.Partition old, Metapb.Partition partition) { log.info("onPartitionChanged partition: {}", partition); if (old != null && old.getState() != partition.getState()) { - // 状态改变，重置图的状态 Metapb.PartitionState state = Metapb.PartitionState.PState_Normal; for (Metapb.Partition pt : partitionMeta.getPartitions(partition.getGraphName())) { if (pt.getState().getNumber() > state.getNumber()) { @@ -1491,7 +1501,7 @@ protected void onPartitionRemoved(Metapb.Partition partition) { } /** - * PD的leader发生改变，需要重新加载数据 + * The leader of the PD has changed and the data needs to be reloaded */ @Override public void onRaftLeaderChanged() { @@ -1503,31 +1513,17 @@ public void onRaftLeaderChanged() { } } - /** - * 分区状态发生改变，需要传播到图、集群 - * - * @param graph - * @param partId - * @param state - */ public void onPartitionStateChanged(String graph, int partId, Metapb.PartitionState state) throws PDException { updatePartitionState(graph, partId, state); } - /** - * Shard状态发生改变，需要传播到分区、图、集群 - * - * @param graph - * @param partId - * @param state - */ public void onShardStateChanged(String graph, int partId, Metapb.PartitionState state) { } /** - * 发送rocksdb compaction 消息 + * Send rocksdb compaction message * * @param partId * @param tableName diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionStatusListener.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionStatusListener.java index fea0ce35d9..690b603622 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionStatusListener.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionStatusListener.java @@ -20,7 +20,7 @@ import org.apache.hugegraph.pd.grpc.Metapb; /** - * 分区状态监听 + * Partition status listeners */ public interface PartitionStatusListener { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java index b755326340..50d810a516 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java @@ -48,14 +48,13 @@ import lombok.extern.slf4j.Slf4j; /** - * HgStore注册、保活管理类 + * Hg Store registration and keep-alive management */ @Slf4j public class StoreNodeService { private static final Long STORE_HEART_BEAT_INTERVAL = 30000L; private static final String graphSpaceConfPrefix = "HUGEGRAPH/hg/GRAPHSPACE/CONF/"; - // Store状态监听 private final List statusListeners; private final List shardGroupStatusListeners; private final StoreInfoMeta storeInfoMeta; @@ -96,7 +95,6 @@ public void init(PartitionService partitionService) { @Override public void onPartitionChanged(Metapb.Partition old, Metapb.Partition partition) { if (old != null && old.getState() != partition.getState()) { - // 状态改变，重置集群状态 try { List partitions = partitionService.getPartitionById(partition.getId()); @@ -128,7 +126,7 @@ public void onPartitionRemoved(Metapb.Partition partition) { } /** - * 集群是否准备就绪 + * Whether the cluster is ready or not * * @return */ @@ -138,36 +136,37 @@ public boolean isOK() { } /** - * Store注册，记录Store的ip地址，首次注册需要生成store_ID + * Store registration, record the IP address of the Store, and the first registration needs + * to generate a store_ID * * @param store */ public Metapb.Store register(Metapb.Store store) throws PDException { if (store.getId() == 0) { - // 初始注册，生成新id，保证Id不重复。 + // Initial registration, generate a new ID, and ensure that the ID is not duplicated. store = newStoreNode(store); } if (!storeInfoMeta.storeExists(store.getId())) { log.error("Store id {} does not belong to this PD, address = {}", store.getId(), store.getAddress()); - // storeId不存在，抛出异常 + // storeId does not exist, an exception is thrown throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE, String.format("Store id %d doest not exist.", store.getId())); } - // 如果store状态为Tombstone拒绝注册。 + // If the store status is Tombstone, the registration is denied. Metapb.Store lastStore = storeInfoMeta.getStore(store.getId()); if (lastStore.getState() == Metapb.StoreState.Tombstone) { log.error("Store id {} has been removed, Please reinitialize, address = {}", store.getId(), store.getAddress()); - // storeId不存在，抛出异常 + // storeId does not exist, an exception is thrown throw new PDException(Pdpb.ErrorType.STORE_HAS_BEEN_REMOVED_VALUE, String.format("Store id %d has been removed. %s", store.getId(), store.getAddress())); } - // offline或者up，或者在初始激活列表中，自动上线 + // offline or up, or in the initial activation list, go live automatically Metapb.StoreState storeState = lastStore.getState(); if (storeState == Metapb.StoreState.Offline || storeState == Metapb.StoreState.Up || inInitialStoreList(store)) { @@ -189,33 +188,34 @@ public Metapb.Store register(Metapb.Store store) throws PDException { long current = System.currentTimeMillis(); boolean raftChanged = false; - // 上线状态的Raft Address 发生了变更 + // On-line status Raft Address there has been a change if (!Objects.equals(lastStore.getRaftAddress(), store.getRaftAddress()) && storeState == Metapb.StoreState.Up) { - // 时间间隔太短，而且raft有变更，则认为是无效的store + // If the time interval is too short and the raft changes, it is considered an + // invalid store if (current - lastStore.getLastHeartbeat() < STORE_HEART_BEAT_INTERVAL * 0.8) { throw new PDException(Pdpb.ErrorType.STORE_PROHIBIT_DUPLICATE_VALUE, String.format("Store id %d may be duplicate. addr: %s", store.getId(), store.getAddress())); } else if (current - lastStore.getLastHeartbeat() > STORE_HEART_BEAT_INTERVAL * 1.2) { - // 认为发生了变更 + // It is considered that a change has occurred raftChanged = true; } else { - // 等待下次注册 + // Wait for the next registration return Metapb.Store.newBuilder(store).setId(0L).build(); } } - // 存储store信息 + // Store information storeInfoMeta.updateStore(store); if (storeState == Metapb.StoreState.Up) { - // 更新store 活跃状态 + // Update the store active status storeInfoMeta.keepStoreAlive(store); onStoreStatusChanged(store, Metapb.StoreState.Offline, Metapb.StoreState.Up); checkStoreStatus(); } - // 等store信息保存后，再发送变更 + // Wait for the store information to be saved before sending the changes if (raftChanged) { onStoreRaftAddressChanged(store); } @@ -229,7 +229,7 @@ private boolean inInitialStoreList(Metapb.Store store) { } /** - * 产生一个新的store对象 + * Creates a new store object * * @param store * @return @@ -249,7 +249,7 @@ private synchronized Metapb.Store newStoreNode(Metapb.Store store) throws PDExce } /** - * 根据store_id返回Store信息 + * Returns Store information based on store_id * * @param id * @return @@ -265,7 +265,7 @@ public Metapb.Store getStore(long id) throws PDException { } /** - * 更新Store信息，检测Store状态的变化，通知到Hugestore + * Update the store information, detect the change of store status, and notify Hugestore */ public synchronized Metapb.Store updateStore(Metapb.Store store) throws PDException { log.info("updateStore storeId: {}, address: {}, state: {}", store.getId(), @@ -290,10 +290,10 @@ public synchronized Metapb.Store updateStore(Metapb.Store store) throws PDExcept storeInfoMeta.updateStore(store); if (store.getState() != Metapb.StoreState.Unknown && store.getState() != lastStore.getState()) { - // 如果希望将store下线 + // If you want to take the store offline if (store.getState() == Metapb.StoreState.Exiting) { if (lastStore.getState() == Metapb.StoreState.Exiting) { - //如果已经是下线中的状态，则不作进一步处理 + // If it is already in the offline state, no further processing will be made return lastStore; } @@ -302,19 +302,23 @@ public synchronized Metapb.Store updateStore(Metapb.Store store) throws PDExcept activeStores.forEach(s -> { storeMap.put(s.getId(), s); }); - //如果store已经离线，直接从活跃中删除，如果store在线，暂时不从活跃中删除，等把状态置成Tombstone的时候再删除 + // If the store is offline, delete it directly from active, and if the store is + // online, temporarily delete it from active, and then delete it when the status + // is set to Tombstone if (!storeMap.containsKey(store.getId())) { log.info("updateStore removeActiveStores store {}", store.getId()); storeInfoMeta.removeActiveStore(store); } storeTurnoff(store); - } else if (store.getState() == Metapb.StoreState.Offline) { //监控到store已经离线，从活跃中删除 + } else if (store.getState() == Metapb.StoreState.Offline) { + // Monitor that the store has gone offline and is removed from the active storeInfoMeta.removeActiveStore(store); } else if (store.getState() == Metapb.StoreState.Tombstone) { - // 状态发生改变，Store关机，修改shardGroup，进行副本迁移 + // When the status changes, the store is shut down, the shardGroup is modified, + // and the replica is migrated log.info("updateStore removeActiveStores store {}", store.getId()); storeInfoMeta.removeActiveStore(store); - // 存储下线 + // Storage goes offline storeTurnoff(store); } else if (store.getState() == Metapb.StoreState.Up) { storeInfoMeta.keepStoreAlive(store); @@ -326,13 +330,13 @@ public synchronized Metapb.Store updateStore(Metapb.Store store) throws PDExcept } /** - * store被关机，重新分配shardGroup的shard + * The shard of the shardGroup is reassigned * * @param store * @throws PDException */ public synchronized void storeTurnoff(Metapb.Store store) throws PDException { - // 遍历ShardGroup，重新分配shard + // Traverse ShardGroup,redistribution for (Metapb.ShardGroup group : getShardGroupsByStore(store.getId())) { Metapb.ShardGroup.Builder builder = Metapb.ShardGroup.newBuilder(group); builder.clearShards(); @@ -346,7 +350,8 @@ public synchronized void storeTurnoff(Metapb.Store store) throws PDException { } /** - * 根据图名返回stores信息，如果graphName为空，返回所有store信息 + * Returns stores information based on the graph name, and if graphName is empty, all store + * information is returned * * @throws PDException */ @@ -391,7 +396,7 @@ public List getShardGroupsByStore(long storeId) throws PDExce } /** - * 返回活跃的store + * Returns the active store * * @param graphName * @return @@ -420,16 +425,19 @@ public long removeStore(Long storeId) throws PDException { } /** - * 给partition分配store，根据图的配置，决定分配几个peer - * 分配完所有的shards，保存ShardGroup对象（store不变动，只执行一次） + * todo : New logic + * Assign a store to the partition and decide how many peers to allocate according to the + * configuration of the graph + * After allocating all the shards, save the ShardGroup object (store does not change, only + * executes once) */ public synchronized List allocShards(Metapb.Graph graph, int partId) throws PDException { - // 多图共用raft分组，因此分配shard只依赖partitionId. - // 图根据数据大小可以设置分区的数量，但总数不能超过raft分组数量 + // Multiple graphs share raft grouping, so assigning shard only depends on partitionId. + // The number of partitions can be set based on the size of the data, but the total + // number cannot exceed the number of raft groups if (storeInfoMeta.getShardGroup(partId) == null) { - // 获取活跃的store key - // 根据 partionID计算store + // Get active store key List stores = storeInfoMeta.getActiveStores(); if (stores.size() == 0) { @@ -445,17 +453,18 @@ public synchronized List allocShards(Metapb.Graph graph, int partI int shardCount = pdConfig.getPartition().getShardCount(); shardCount = Math.min(shardCount, stores.size()); - //两个shard无法选出leader - // 不能为0 + // Two shards could not elect a leader + // It cannot be 0 if (shardCount == 2 || shardCount < 1) { shardCount = 1; } - // 一次创建完所有的ShardGroup，保证初始的groupID有序，方便人工阅读 + // All ShardGroups are created at one time to ensure that the initial groupIDs are + // orderly and easy for humans to read for (int groupId = 0; groupId < pdConfig.getConfigService().getPartitionCount(); groupId++) { - int storeIdx = groupId % stores.size(); //store分配规则，简化为取模 + int storeIdx = groupId % stores.size(); // Assignment rules, simplified to modulo List shards = new ArrayList<>(); for (int i = 0; i < shardCount; i++) { Metapb.Shard shard = @@ -464,7 +473,7 @@ public synchronized List allocShards(Metapb.Graph graph, int partI Metapb.ShardRole.Follower) // .build(); shards.add(shard); - storeIdx = (storeIdx + 1) >= stores.size() ? 0 : ++storeIdx; // 顺序选择 + storeIdx = (storeIdx + 1) >= stores.size() ? 0 : ++storeIdx; // Sequential } Metapb.ShardGroup group = Metapb.ShardGroup.newBuilder() @@ -484,8 +493,8 @@ public synchronized List allocShards(Metapb.Graph graph, int partI } /** - * 根据graph的shard_count，重新分配shard - * 发送变更change shard指令 + * Based on the shard_count of the graph, reallocate shards + * Send change shard */ public synchronized List reallocShards(Metapb.ShardGroup shardGroup) throws PDException { @@ -505,8 +514,8 @@ public synchronized List reallocShards(Metapb.ShardGroup shardGrou int shardCount = pdConfig.getPartition().getShardCount(); shardCount = Math.min(shardCount, stores.size()); if (shardCount == 2 || shardCount < 1) { - // 两个shard无法选出leader - // 不能为0 + // Two shards could not elect a leader + // It cannot be 0 shardCount = 1; } @@ -514,12 +523,12 @@ public synchronized List reallocShards(Metapb.ShardGroup shardGrou shards.addAll(shardGroup.getShardsList()); if (shardCount > shards.size()) { - // 需要增加shard + // Need to add shards log.info("reallocShards ShardGroup {}, add shards from {} to {}", shardGroup.getId(), shards.size(), shardCount); - int storeIdx = shardGroup.getId() % stores.size(); //store分配规则，简化为取模 + int storeIdx = shardGroup.getId() % stores.size(); for (int addCount = shardCount - shards.size(); addCount > 0; ) { - // 检查是否已经存在 + // Check if it already exists if (!isStoreInShards(shards, stores.get(storeIdx).getId())) { Metapb.Shard shard = Metapb.Shard.newBuilder() .setStoreId(stores.get(storeIdx).getId()) @@ -527,10 +536,10 @@ public synchronized List reallocShards(Metapb.ShardGroup shardGrou shards.add(shard); addCount--; } - storeIdx = (storeIdx + 1) >= stores.size() ? 0 : ++storeIdx; // 顺序选择 + storeIdx = (storeIdx + 1) >= stores.size() ? 0 : ++storeIdx; } } else if (shardCount < shards.size()) { - // 需要减shard + // Need to reduce shard log.info("reallocShards ShardGroup {}, remove shards from {} to {}", shardGroup.getId(), shards.size(), shardCount); @@ -566,7 +575,7 @@ public synchronized List reallocShards(Metapb.ShardGroup shardGrou } /** - * 根据partition的数量，分配group shard + * According to the number of partitions，distribute group shard * * @param groups list of (partition id, count) * @return total groups @@ -574,7 +583,7 @@ public synchronized List reallocShards(Metapb.ShardGroup shardGrou public synchronized int splitShardGroups(List> groups) throws PDException { int sum = groups.stream().map(pair -> pair.getValue()).reduce(0, Integer::sum); - // shard group 太大 + // shard group is too big if (sum > getActiveStores().size() * pdConfig.getPartition().getMaxShardsPerStore()) { throw new PDException(Pdpb.ErrorType.Too_Many_Partitions_Per_Store_VALUE, "can't satisfy target shard group count"); @@ -586,8 +595,9 @@ public synchronized int splitShardGroups(List> groups) } /** - * 分配shard group，为分裂做准备 + * Alloc shard group, prepare for the split * + * @param * @return true * @throws PDException */ @@ -639,10 +649,10 @@ public synchronized Metapb.ShardGroup updateShardGroup(int groupId, List shards) throws PDException { @@ -665,7 +675,7 @@ public void shardGroupOp(int groupId, List shards) throws PDExcept } /** - * 删除 shard group + * Delete shard group * * @param groupId shard group id */ @@ -677,7 +687,7 @@ public synchronized void deleteShardGroup(int groupId) throws PDException { onShardGroupStatusChanged(group, null); - // 修正store的分区数. (分区合并导致) + // Fix the number of partitions for the store. (Result from partition merge) var shardGroups = getShardGroups(); if (shardGroups != null) { var count1 = pdConfig.getConfigService().getPDConfig().getPartitionCount(); @@ -699,7 +709,7 @@ public synchronized void updateShardGroupState(int groupId, Metapb.PartitionStat } /** - * 接收Store的心跳 + * Receive the heartbeat of the Store * * @param storeStats * @throws PDException @@ -708,7 +718,7 @@ public Metapb.ClusterStats heartBeat(Metapb.StoreStats storeStats) throws PDExce this.storeInfoMeta.updateStoreStats(storeStats); Metapb.Store lastStore = this.getStore(storeStats.getStoreId()); if (lastStore == null) { - //store不存在 + // store does not exist throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE, String.format("Store id %d does not exist.", storeStats.getStoreId())); @@ -720,14 +730,16 @@ public Metapb.ClusterStats heartBeat(Metapb.StoreStats storeStats) throws PDExce storeStats.getStoreId())); } Metapb.Store nowStore; - // 如果正在做store下线操作 + // If you are going to take the store offline if (lastStore.getState() == Metapb.StoreState.Exiting) { List activeStores = this.getActiveStores(); Map storeMap = new HashMap<>(); activeStores.forEach(store -> { storeMap.put(store.getId(), store); }); - // 下线的store的分区为0，说明已经迁移完毕，可以下线，如果非0，则迁移还在进行，需要等待 + // If the partition of the offline store is 0, it means that the migration has been + // completed and can be taken offline, if it is not 0, the migration is still in + // progress and you need to wait if (storeStats.getPartitionCount() > 0 && storeMap.containsKey(storeStats.getStoreId())) { nowStore = Metapb.Store.newBuilder(lastStore) @@ -799,9 +811,9 @@ public Metapb.ClusterStats getClusterStats() { } /** - * 检查集群健康状态 - * 活跃机器数是否大于最小阈值 - * 分区shard在线数已否过半 * + * Check the cluster health status + * Whether the number of active machines is greater than the minimum threshold + * The number of partition shards online has exceeded half */ public synchronized void checkStoreStatus() { Metapb.ClusterStats.Builder builder = Metapb.ClusterStats.newBuilder() @@ -821,7 +833,7 @@ public synchronized void checkStoreStatus() { }); if (builder.getState() == Metapb.ClusterState.Cluster_OK) { - // 检查每个分区的在线shard数量是否大于半数 + // Check whether the number of online shards for each partition is greater than half for (Metapb.ShardGroup group : this.getShardGroups()) { int count = 0; for (Metapb.Shard shard : group.getShardsList()) { @@ -881,9 +893,10 @@ protected void onShardGroupOp(Metapb.ShardGroup shardGroup) { } /** - * 检查当前store是否可下线 - * 活跃机器数小于等于最小阈值，不可下线 - * 分区shard在线数不超过半数， 不可下线 + * Check whether the current store can be discontinued + * If the number of active machines is less than or equal to the minimum threshold, they + * cannot be taken offline + * If the number of shards in the partition is not more than half, it cannot be offline */ public boolean checkStoreCanOffline(Metapb.Store currentStore) { try { @@ -900,7 +913,7 @@ public boolean checkStoreCanOffline(Metapb.Store currentStore) { return false; } - // 检查每个分区的在线shard数量是否大于半数 + // Check whether the number of online shards for each partition is greater than half for (Metapb.ShardGroup group : this.getShardGroups()) { int count = 0; for (Metapb.Shard shard : group.getShardsList()) { @@ -920,7 +933,7 @@ public boolean checkStoreCanOffline(Metapb.Store currentStore) { } /** - * 对store上的对rocksdb进行compaction + * Compaction on rocksdb on the store * * @param groupId * @param tableName @@ -929,9 +942,9 @@ public boolean checkStoreCanOffline(Metapb.Store currentStore) { public synchronized void shardGroupsDbCompaction(int groupId, String tableName) throws PDException { - // 通知所有的store，对rocksdb进行compaction + // Notify all stores to compaction rocksdb partitionService.fireDbCompaction(groupId, tableName); - // TODO 异常怎么处理？ + // TODO How to deal with exceptions? } public Map getQuota() throws PDException { @@ -1037,7 +1050,7 @@ public StoreInfoMeta getStoreInfoMeta() { } /** - * 获得分区的Leader + * Get the leader of the partition * * @param partition * @param initIdx diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java index 889e5a0234..9e933a6368 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java @@ -44,18 +44,21 @@ import lombok.extern.slf4j.Slf4j; /** - * 任务调度服务，定时检查Store、资源、分区的状态，及时迁移数据，错误节点 - * 1、监测Store是否离线 - * 2、监测Partition的副本是否正确 - * 3、监测Partition的工作模式是否正确 - * 4、监测Partition是否需要分裂,监测分裂是否完成 + * The task scheduling service checks the status of stores, resources, and partitions on a + * regular basis, migrates data in a timely manner, and errors are on nodes + * 1. Monitor whether the store is offline + * 2. Check whether the replica of the partition is correct + * 3. Check whether the working mode of the partition is correct + * 4. Monitor whether the partition needs to be split and whether the split is completed */ @Slf4j public class TaskScheduleService { private static final String BALANCE_SHARD_KEY = "BALANCE_SHARD_KEY"; - private final long TurnOffAndBalanceInterval = 30 * 60 * 1000; //机器下线30后才能进行动态平衡 - private final long BalanceLeaderInterval = 30 * 1000; // leader平衡时间间隔 + // The dynamic balancing can only be carried out after the machine is offline for 30 minutes + private final long TurnOffAndBalanceInterval = 30 * 60 * 1000; + // leader balances the time interval + private final long BalanceLeaderInterval = 30 * 1000; private final PDConfig pdConfig; private final long clusterStartTime; // private final StoreNodeService storeService; @@ -65,14 +68,12 @@ public class TaskScheduleService { private final StoreMonitorDataService storeMonitorDataService; private final KvService kvService; private final LogService logService; - // 先按照value排序，再按照key排序 private final Comparator> kvPairComparatorAsc = (o1, o2) -> { if (o1.getValue() == o2.getValue()) { return o1.getKey().compareTo(o2.getKey()); } return o1.getValue().compareTo(o2.getValue()); }; - // 先按照value排序(倒序)，再按照key排序(升序） private final Comparator> kvPairComparatorDesc = (o1, o2) -> { if (o1.getValue() == o2.getValue()) { return o2.getKey().compareTo(o1.getKey()); @@ -157,7 +158,7 @@ public void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old, if (status == Metapb.StoreState.Up) { executor.schedule(() -> { - try { //store 上线后延时1分钟进行leader平衡 + try { balancePartitionLeader(false); } catch (PDException e) { log.error("exception {}", e); @@ -190,7 +191,7 @@ private boolean isLeader() { } /** - * 巡查所有的store，检查是否在线，存储空间是否充足 + * Inspect all stores to see if they are online and have enough storage space */ public List patrolStores() throws PDException { if (!isLeader()) { @@ -198,7 +199,7 @@ public List patrolStores() throws PDException { } List changedStores = new ArrayList<>(); - // 检查store在线状态 + // Check your store online status List stores = storeService.getStores(""); Map activeStores = storeService.getActiveStores("") .stream().collect( @@ -208,7 +209,7 @@ public List patrolStores() throws PDException { if ((store.getState() == Metapb.StoreState.Up || store.getState() == Metapb.StoreState.Unknown) && !activeStores.containsKey(store.getId())) { - // 不在线，修改状态为离线 + // If you are not online, the modification status is offline changeStore = Metapb.Store.newBuilder(store) .setState(Metapb.StoreState.Offline) .build(); @@ -220,8 +221,8 @@ public List patrolStores() throws PDException { pdConfig.getStore().getMaxDownTime() * 1000) && (System.currentTimeMillis() - clusterStartTime > pdConfig.getStore().getMaxDownTime() * 1000))) { - //手工修改为下线或者离线达到时长 - // 修改状态为关机, 增加 checkStoreCanOffline 检测 + // Manually change the parameter to Offline or Offline Duration + // Modify the status to shut down and increase checkStoreCanOffline detect if (storeService.checkStoreCanOffline(store)) { changeStore = Metapb.Store.newBuilder(store) .setState(Metapb.StoreState.Tombstone).build(); @@ -239,22 +240,22 @@ public List patrolStores() throws PDException { } /** - * 巡查所有的分区，检查副本数是否正确 + * Inspect all partitions to check whether the number of replicas is correct and the number + * of replicas in the shard group */ public List patrolPartitions() throws PDException { if (!isLeader()) { return null; } - // 副本数不一致，重新分配副本 + // If the number of replicas is inconsistent, reallocate replicas for (Metapb.ShardGroup group : storeService.getShardGroups()) { if (group.getShardsCount() != pdConfig.getPartition().getShardCount()) { storeService.reallocShards(group); - // 避免后面的 balance partition shard 马上执行. kvService.put(BALANCE_SHARD_KEY, "DOING", 180 * 1000); } } - //检查shard是否在线。 + // Check if the shard is online. Map tombStores = storeService.getTombStores().stream().collect( Collectors.toMap(Metapb.Store::getId, t -> t)); @@ -277,8 +278,8 @@ public List patrolPartitions() throws PDException { } /** - * 在Store之间平衡分区的数量 - * 机器转为UP半小时后才能进行动态平衡 + * Balance the number of partitions between stores + * It takes half an hour for the machine to turn to UP before it can be dynamically balanced */ public synchronized Map> balancePartitionShard() throws PDException { @@ -289,7 +290,7 @@ public synchronized Map> balancePartitionShard() thr } if (System.currentTimeMillis() - lastStoreTurnoffTime < TurnOffAndBalanceInterval) { - return null;//机器下线半小时后才能进行动态平衡 + return null; } int activeStores = storeService.getActiveStores().size(); @@ -298,8 +299,6 @@ public synchronized Map> balancePartitionShard() thr return null; } - // 避免频繁调用. (当改变副本数，需要调整shard list，此时又需要平衡分区）会发送重复的指令。造成结果不可预料。 - // 严重会删除掉分区. if (Objects.equals(kvService.get(BALANCE_SHARD_KEY), "DOING")) { return null; } @@ -309,20 +308,18 @@ public synchronized Map> balancePartitionShard() thr int averageCount = totalShards / activeStores; int remainder = totalShards % activeStores; - // 统计每个store上分区, StoreId ->PartitionID, ShardRole + // Count the partitions on each store, StoreId -> PartitionID, ShardRole Map> partitionMap = new HashMap<>(); storeService.getActiveStores().forEach(store -> { partitionMap.put(store.getId(), new HashMap<>()); }); - // 如果是leaner 说明迁移正在进行，不要重复提交任务 AtomicReference isLeaner = new AtomicReference<>(false); partitionService.getPartitions().forEach(partition -> { try { storeService.getShardList(partition.getId()).forEach(shard -> { Long storeId = shard.getStoreId(); - // 判断每个shard为leaner或者状态非正常状态 if (shard.getRole() == Metapb.ShardRole.Learner || partition.getState() != Metapb.PartitionState.PState_Normal) { isLeaner.set(true); @@ -342,24 +339,25 @@ public synchronized Map> balancePartitionShard() thr return null; } - // 按照shard数量由高到低排序store + // According to shard sort the quantity from highest to lowest List> sortedList = new ArrayList<>(); partitionMap.forEach((storeId, shards) -> { sortedList.add(new KVPair(storeId, shards.size())); }); - // 由大到小排序的list sortedList.sort(((o1, o2) -> o2.getValue().compareTo(o1.getValue()))); - // 最大堆 + // The largest heap, moved in store -> shard count PriorityQueue> maxHeap = new PriorityQueue<>(sortedList.size(), (o1, o2) -> o2.getValue() .compareTo( o1.getValue())); - // 各个副本的 committedIndex + // of individual copies committedIndex Map> committedIndexMap = partitionService.getCommittedIndexStats(); - // 分区ID --> 源StoreID,目标StoreID + // Partition ID -->source StoreID, target StoreID Map> movedPartitions = new HashMap<>(); - // 移除多余的shard, 按照shards由多到少的顺序遍历store，余数remainder优先给shards多的store分配，减少迁移的概率 + // Remove redundant shards, traverse the stores in the order of shards from most to + // least, and the remainder is allocated to the store with more shards first, reducing + // the probability of migration for (int index = 0; index < sortedList.size(); index++) { long storeId = sortedList.get(index).getKey(); if (!partitionMap.containsKey(storeId)) { @@ -368,7 +366,8 @@ public synchronized Map> balancePartitionShard() thr } Map shards = partitionMap.get(storeId); int targetCount = index < remainder ? averageCount + 1 : averageCount; - // 移除多余的shard, 添加源StoreID. 非Leader，并且该分区唯一 + // Remove the redundant shards and add the source StoreID. is not a leader, and the + // partition is unique if (shards.size() > targetCount) { int movedCount = shards.size() - targetCount; log.info( @@ -420,7 +419,7 @@ public synchronized Map> balancePartitionShard() thr if (partitionMap.containsKey(destStoreId)) { destContains = partitionMap.get(destStoreId).containsKey(partitionId); } - // 如果目的store已经包含了该partition，则取一下store + // If the destination store already contains the partition, take the store if (!destContains) { moveEntry.getValue().setValue(pair.getKey()); log.info( @@ -442,9 +441,9 @@ public synchronized Map> balancePartitionShard() thr kvService.put(BALANCE_SHARD_KEY, "DOING", 180 * 1000); - // 开始迁移 + // Start the migration movedPartitions.forEach((partId, storePair) -> { - // 源和目标storeID都不为0 + // Neither the source nor destination storeID is 0 if (storePair.getKey() > 0 && storePair.getValue() > 0) { partitionService.movePartitionsShard(partId, storePair.getKey(), storePair.getValue()); @@ -457,7 +456,7 @@ public synchronized Map> balancePartitionShard() thr } /** - * 在Store之间平衡分区的Leader的数量 + * Balance the number of leaders of partitions between stores */ public synchronized Map balancePartitionLeader(boolean immediately) throws PDException { @@ -475,13 +474,12 @@ public synchronized Map balancePartitionLeader(boolean immediatel List shardGroups = storeService.getShardGroups(); - // 分裂或者缩容任务的时候，退出 + // When a task is split or scaled-in, it is exited var taskMeta = storeService.getTaskInfoMeta(); if (taskMeta.hasSplitTaskDoing() || taskMeta.hasMoveTaskDoing()) { throw new PDException(1001, "split or combine task is processing, please try later!"); } - // 数据迁移的时候，退出 if (Objects.equals(kvService.get(BALANCE_SHARD_KEY), "DOING")) { throw new PDException(1001, "balance shard is processing, please try later!"); } @@ -502,7 +500,6 @@ public synchronized Map balancePartitionLeader(boolean immediatel log.info("balancePartitionLeader, shard group size: {}, by store: {}", shardGroups.size(), storeShardCount); - // 按照 target count， store id稳定排序 PriorityQueue> targetCount = new PriorityQueue<>(kvPairComparatorDesc); @@ -520,7 +517,6 @@ public synchronized Map balancePartitionLeader(boolean immediatel targetCount.add(new KVPair<>(sortedGroups.get(i).getKey(), v)); sum += v; } - // 最后一个, 除不尽的情况，保证总数正确 targetCount.add(new KVPair<>(sortedGroups.get(sortedGroups.size() - 1).getKey(), shardGroups.size() - sum)); log.info("target count: {}", targetCount); @@ -529,7 +525,8 @@ public synchronized Map balancePartitionLeader(boolean immediatel var map = group.getShardsList().stream() .collect(Collectors.toMap(Metapb.Shard::getStoreId, shard -> shard)); var tmpList = new ArrayList>(); - // store比较多的情况，可能不包含对应的store id. 则先将不符合的store保存到临时列表，直到找到一个合适的store + // If there are many stores, they may not contain the corresponding store ID. Save + // the non-compliant stores to the temporary list until you find a suitable store while (!targetCount.isEmpty()) { var pair = targetCount.poll(); var storeId = pair.getKey(); @@ -549,7 +546,7 @@ public synchronized Map balancePartitionLeader(boolean immediatel pair.setValue(pair.getValue() - 1); tmpList.add(pair); } - // 找到了，则处理完成 + // If it is found, the processing is complete break; } else { tmpList.add(pair); @@ -574,14 +571,13 @@ private long getMaxIndexGap(Map> committedIndexMap, int shardMap.forEach((storeId, committedIndex) -> { sortedList.add(committedIndex); }); - // 由大到小排序的list sortedList.sort(Comparator.reverseOrder()); maxGap = sortedList.get(0) - sortedList.get(sortedList.size() - 1); return maxGap; } /** - * 执行分区分裂，分为自动分裂和手工分裂 + * Perform partition splitting, which is divided into automatic splitting and manual splitting * * @return * @throws PDException @@ -602,9 +598,11 @@ public List splitPartition( } /** - * 自动进行分区分裂，每个store达到最大分区数量 - * 执行条件 - * 分裂后每台机器分区数量少于partition.max-partitions-per-store + * Partition splitting is performed automatically, and each store reaches the maximum number + * of partitions + * execution conditions + * The number of partitions per machine after the split is less than partition + * .max-partitions-per-store * * @throws PDException */ @@ -623,11 +621,7 @@ public List autoSplitPartition() throws PDException { } } - //For TEST - // pdConfig.getPartition().setMaxShardsPerStore(pdConfig.getPartition() - // .getMaxShardsPerStore()*2); - - // 计算集群能能支持的最大split count + // The maximum split count that a compute cluster can support int splitCount = pdConfig.getPartition().getMaxShardsPerStore() * storeService.getActiveStores().size() / (storeService.getShardGroups().size() * @@ -640,12 +634,12 @@ public List autoSplitPartition() throws PDException { + pdConfig.getPartition().getMaxShardsPerStore()); } - // 每store未达最大分区数，进行分裂 + // If the maximum number of partitions per store is not reached, it will be split log.info("Start to split partitions..., split count = {}", splitCount); - // 设置集群状态为下线 + // Set the cluster status to Offline storeService.updateClusterStatus(Metapb.ClusterState.Cluster_Offline); - // 修改默认分区数量 + // Modify the default number of partitions // pdConfig.getConfigService().setPartitionCount(storeService.getShardGroups().size() * // splitCount); @@ -658,8 +652,9 @@ public List autoSplitPartition() throws PDException { } /** - * Store汇报任务状态 - * 分区状态发生改变，重新计算分区所在的ShardGroup、图和整个集群的状态 + * Store reports the status of the task + * The state of the partition changes, and the state of the ShardGroup, graph, and the entire + * cluster where the partition resides * * @param task */ @@ -684,7 +679,7 @@ public void reportTask(MetaTask.Task task) { } /** - * 对rocksdb进行compaction + * Compaction on rocksdb * * @throws PDException */ @@ -702,39 +697,44 @@ public Boolean dbCompaction(String tableName) throws PDException { } /** - * 判断是否能把一个store的分区全部迁出，给出判断结果和迁移方案 + * Determine whether all partitions of a store can be migrated out, and give the judgment + * result and migration plan */ public Map canAllPartitionsMovedOut(Metapb.Store sourceStore) throws PDException { if (!isLeader()) { return null; } - // 分析一个store上面的分区是否可以完全迁出 + // Analyze whether the partition on a store can be completely checked out Map resultMap = new HashMap<>(); - // 定义对象用于保存源store上面的分区 StoreId ->PartitionID, ShardRole + // The definition object is used to hold the partition above the source store StoreId + // ->PartitionID, ShardRole Map> sourcePartitionMap = new HashMap<>(); sourcePartitionMap.put(sourceStore.getId(), new HashMap<>()); - // 定义对象用于保存其他活跃store上面的分区 StoreId ->PartitionID, ShardRole + // The definition object is used to hold the partition above the other active stores + // StoreId ->PartitionID, ShardRole Map> otherPartitionMap = new HashMap<>(); - Map availableDiskSpace = new HashMap<>(); // 每个store剩余的磁盘空间 - Map partitionDataSize = new HashMap<>(); // 记录待迁移的分区的数据量 + // The amount of disk space remaining for each store + Map availableDiskSpace = new HashMap<>(); + // Record the amount of data in the partition to be migrated + Map partitionDataSize = new HashMap<>(); storeService.getActiveStores().forEach(store -> { if (store.getId() != sourceStore.getId()) { otherPartitionMap.put(store.getId(), new HashMap<>()); - // 记录其他store的剩余的磁盘空间, 单位为Byte + // Records the remaining disk space of other stores, in bytes availableDiskSpace.put(store.getId(), store.getStats().getAvailable()); } else { resultMap.put("current_store_is_online", true); } }); - // 统计待迁移的分区的数据大小 (从storeStats中统计，单位为KB) + // Count the size of the partition to be migrated (from storeStats in KB) for (Metapb.GraphStats graphStats : sourceStore.getStats().getGraphStatsList()) { partitionDataSize.put(graphStats.getPartitionId(), partitionDataSize.getOrDefault(graphStats.getPartitionId(), 0L) + graphStats.getApproximateSize()); } - // 给sourcePartitionMap 和 otherPartitionMap赋值 + // Assign values to sourcePartitionMap and otherPartitionMap partitionService.getPartitions().forEach(partition -> { try { storeService.getShardList(partition.getId()).forEach(shard -> { @@ -752,13 +752,14 @@ public Map canAllPartitionsMovedOut(Metapb.Store sourceStore) th throw new RuntimeException(e); } }); - // 统计待移除的分区：即源store上面的所有分区 + // Count the partitions to be removed: all partitions on the source store Map> movedPartitions = new HashMap<>(); for (Map.Entry entry : sourcePartitionMap.get( sourceStore.getId()).entrySet()) { movedPartitions.put(entry.getKey(), new KVPair<>(sourceStore.getId(), 0L)); } - // 统计其他store的分区数量, 用小顶堆保存，以便始终把分区数量较少的store优先考虑 + // Count the number of partitions of other stores and save them with a small top heap, so + // that stores with fewer partitions are always prioritized PriorityQueue> minHeap = new PriorityQueue<>(otherPartitionMap.size(), (o1, o2) -> o1.getValue() .compareTo( @@ -766,24 +767,28 @@ public Map canAllPartitionsMovedOut(Metapb.Store sourceStore) th otherPartitionMap.forEach((storeId, shards) -> { minHeap.add(new KVPair(storeId, shards.size())); }); - // 遍历待迁移的分区,优先迁移到分区比较少的store + // Traverse the partitions to be migrated, and prioritize the migration to the store with + // fewer partitions Iterator>> moveIterator = movedPartitions.entrySet().iterator(); while (moveIterator.hasNext()) { Map.Entry> moveEntry = moveIterator.next(); int partitionId = moveEntry.getKey(); - List> tmpList = new ArrayList<>(); // 记录已经弹出优先队列的元素 + // Record the elements that have popped up in the priority + List> tmpList = new ArrayList<>(); while (minHeap.size() > 0) { - KVPair pair = minHeap.poll(); //弹出首个元素 + KVPair pair = minHeap.poll(); // The first element pops up long storeId = pair.getKey(); int partitionCount = pair.getValue(); Map shards = otherPartitionMap.get(storeId); - final int unitRate = 1024; // 平衡不同存储单位的进率 + final int unitRate = 1024; // Balance the feed rate of different storage units if ((!shards.containsKey(partitionId)) && ( availableDiskSpace.getOrDefault(storeId, 0L) / unitRate >= partitionDataSize.getOrDefault(partitionId, 0L))) { - // 如果目标store上面不包含该分区，且目标store剩余空间能容纳该分区，则进行迁移 - moveEntry.getValue().setValue(storeId); //设置移动的目标store + // If the partition is not included on the destination store and the + // remaining space of the destination store can accommodate the partition, + // the migration is performed + moveEntry.getValue().setValue(storeId); // Set the target store for the move log.info("plan to move partition {} to store {}, " + "available disk space {}, current partitionSize:{}", partitionId, @@ -791,12 +796,12 @@ public Map canAllPartitionsMovedOut(Metapb.Store sourceStore) th availableDiskSpace.getOrDefault(storeId, 0L) / unitRate, partitionDataSize.getOrDefault(partitionId, 0L) ); - // 更新该store预期的剩余空间 + // Update the expected remaining space for the store availableDiskSpace.put(storeId, availableDiskSpace.getOrDefault(storeId, 0L) - partitionDataSize.getOrDefault(partitionId, 0L) * unitRate); - // 更新统计变量中该store的分区数量 + // Update the number of partitions for that store in the stat variable partitionCount += 1; pair.setValue(partitionCount); tmpList.add(pair); @@ -807,7 +812,7 @@ public Map canAllPartitionsMovedOut(Metapb.Store sourceStore) th } minHeap.addAll(tmpList); } - //检查是否未存在未分配目标store的分区 + // Check that there are no partitions that don't have a target store assigned List remainPartitions = new ArrayList<>(); movedPartitions.forEach((partId, storePair) -> { if (storePair.getValue() == 0L) { @@ -830,10 +835,10 @@ public Map> movePartitions( if (!isLeader()) { return null; } - // 开始迁移 + // Start the migration log.info("begin move partitions:"); movedPartitions.forEach((partId, storePair) -> { - // 源和目标storeID都不为0 + // Neither the source nor destination storeID is 0 if (storePair.getKey() > 0 && storePair.getValue() > 0) { partitionService.movePartitionsShard(partId, storePair.getKey(), storePair.getValue()); diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java index 6ff66459ef..0478b33da6 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java @@ -33,23 +33,26 @@ import lombok.Data; /** - * PD配置文件 + * PD profile */ @Data @Component public class PDConfig { + // cluster ID @Value("${pd.cluster_id:1}") - private long clusterId; // 集群ID + private long clusterId; + // The patrol task interval @Value("${pd.patrol-interval:300}") - private long patrolInterval = 300; //巡查任务时间间隔 + private long patrolInterval = 300; @Value("${pd.data-path}") private String dataPath; @Value("${pd.initial-store-count:3}") private int minStoreCount; - // 初始store列表，该列表内的store自动激活 + // The initial store list, within which the store is automatically activated + // format: store_addresss, store_address, store_address/group_id, store_address/group_id @Value("${pd.initial-store-list: ''}") private String initialStoreList; @Value("${grpc.host}") @@ -84,8 +87,8 @@ public Map getInitialStoreMap() { } /** - * 初始分区数量 - * Store数量 * 每Store最大副本数 /每分区副本数 + * The initial number of partitions + * Number of Stores * Maximum number of replicas per Store / Number of replicas per partition * * @return */ @@ -144,7 +147,7 @@ public class Raft { private int port; @Value("${pd.cluster_id:1}") - private long clusterId; // 集群ID + private long clusterId; @Value("${grpc.port}") private int grpcPort; @@ -157,7 +160,7 @@ public String getGrpcAddress() { @Configuration public class Store { - // store 心跳超时时间 + // store Heartbeat timeout @Value("${store.keepAlive-timeout:300}") private long keepAliveTimeout = 300; @Value("${store.max-down-time:1800}") @@ -249,11 +252,10 @@ public class Partition { private int totalCount = 0; - // 每个Store最大副本数 + // Maximum number of replicas per Store @Value("${partition.store-max-shard-count:24}") private int maxShardsPerStore = 24; - // 默认分副本数量 @Value("${partition.default-shard-count:3}") private int shardCount = 3; @@ -273,7 +275,8 @@ public void setTotalCount(int totalCount) { @Configuration public class Discovery { - // 客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 + // After the client registers, the maximum number of heartbeats is not reached, and after + // that, the previous registration information will be deleted @Value("${discovery.heartbeat-try-count:3}") private int heartbeatOutTimes = 3; } diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/ConfigMetaStore.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/ConfigMetaStore.java index df332f46b6..edcec9fc85 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/ConfigMetaStore.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/ConfigMetaStore.java @@ -34,7 +34,7 @@ public ConfigMetaStore(PDConfig pdConfig) { } /** - * 更新图空间存储状态信息 + * Update the storage status of the graph space * * @param */ diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java index a986487b2c..661670b8a8 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java @@ -35,7 +35,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 自增id的实现类 + * Implementation class for auto-increment ID. */ @Slf4j public class IdMetaStore extends MetadataRocksDBStore { @@ -46,7 +46,7 @@ public class IdMetaStore extends MetadataRocksDBStore { private static final String CID_DEL_SLOT_PREFIX = "@CID_DEL_SLOT@"; private static final String SEPARATOR = "@"; private static final ConcurrentHashMap SEQUENCES = new ConcurrentHashMap<>(); - private static long CID_DEL_TIMEOUT = 24 * 3600 * 1000; + private static final long CID_DEL_TIMEOUT = 24 * 3600 * 1000; private final long clusterId; public IdMetaStore(PDConfig pdConfig) { @@ -67,7 +67,7 @@ public static byte[] longToBytes(long l) { } /** - * 获取自增id + * Get auto-increment ID. * * @param key * @param delta @@ -108,17 +108,19 @@ public void resetId(String key) throws PDException { } /** - * 在删除name标识的cid的24小时内重复申请同一个name的cid保持同一值 - * 如此设计为了防止缓存的不一致，造成数据错误 + * Within 24 hours of deleting the cid identified by the name, + * repeat applying for the same name's cid to keep the same value. + * This design is to prevent inconsistent caching, causing data errors. * * @param key - * @param name cid 标识 + * @param name cid identifier * @param max * @return * @throws PDException */ public long getCId(String key, String name, long max) throws PDException { - // 检测是否有过期的cid，删除图的频率比较低，此处对性能影响不大 + // Check for expired cids to delete. The frequency of deleting graphs is relatively low, + // so this has little performance impact. byte[] delKeyPrefix = (CID_DEL_SLOT_PREFIX + key + SEPARATOR).getBytes(Charset.defaultCharset()); synchronized (this) { @@ -136,11 +138,11 @@ public long getCId(String key, String name, long max) throws PDException { } }); - // 从延时删除队列恢复Key + // Restore key from delayed deletion queue byte[] cidDelayKey = getCIDDelayKey(key, name); byte[] value = getOne(cidDelayKey); if (value != null) { - // 从延迟删除队列删除 + // Remove from delayed deletion queue remove(cidDelayKey); return ((long[]) deserialize(value))[0]; } else { @@ -150,7 +152,7 @@ public long getCId(String key, String name, long max) throws PDException { } /** - * 添加到删除队列，延后删除 + * Add to the deletion queue for delayed deletion. */ public long delCIdDelay(String key, String name, long cid) throws PDException { byte[] delKey = getCIDDelayKey(key, name); @@ -159,10 +161,12 @@ public long delCIdDelay(String key, String name, long cid) throws PDException { } /** - * 获取自增循环不重复id, 达到上限后从0开始自增 + * Get an auto-incrementing cyclic non-repeating ID. When the upper limit is reached, it + * starts from 0 again. * * @param key - * @param max id上限，达到该值后，重新从0开始自增 + * @param max the upper limit of the ID. After reaching this value, it starts incrementing + * from 0 again. * @return * @throws PDException */ @@ -173,7 +177,7 @@ public long getCId(String key, long max) throws PDException { byte[] bs = getOne(keyBs); long current = bs != null ? bytesToLong(bs) : 0L; long last = current == 0 ? max - 1 : current - 1; - { // 查找一个未使用的cid + { // Find an unused cid List kvs = scanRange(genCIDSlotKey(key, current), genCIDSlotKey(key, max)); for (KV kv : kvs) { if (current == bytesToLong(kv.getValue())) { @@ -219,7 +223,7 @@ private byte[] getCIDDelayKey(String key, String name) { } /** - * 删除一个循环id，释放id值 + * Delete a cyclic ID and release its value. * * @param key * @param cid diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataFactory.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataFactory.java index cc247041cf..673c313dfc 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataFactory.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataFactory.java @@ -24,7 +24,7 @@ import org.apache.hugegraph.pd.store.RaftKVStore; /** - * 存储工厂类，创建相关对象的存储类 + * Storage Factory class to create a storage class for related objects */ public class MetadataFactory { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataRocksDBStore.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataRocksDBStore.java index bf77e41c05..7a12a0afa0 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataRocksDBStore.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataRocksDBStore.java @@ -108,7 +108,6 @@ public void removeWithTTL(byte[] key) throws PDException { @Override public List scanPrefix(byte[] prefix) throws PDException { - //TODO 使用rocksdb 前缀查询 try { return this.store.scanPrefix(prefix); } catch (Exception e) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataStoreBase.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataStoreBase.java index 4cd9e1d364..ae7fd2079c 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataStoreBase.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataStoreBase.java @@ -30,7 +30,7 @@ public abstract class MetadataStoreBase { - // public long timeout = 3; // 请求超时时间，默认三秒 + // public long timeout = 3; public abstract byte[] getOne(byte[] key) throws PDException; @@ -39,9 +39,8 @@ public abstract class MetadataStoreBase { public abstract void put(byte[] key, byte[] value) throws PDException; /** - * 带有过期时间的put + * A put with an expiration time */ - public abstract void putWithTTL(byte[] key, byte[] value, long ttl) throws PDException; @@ -57,7 +56,7 @@ public abstract void putWithTTL(byte[] key, public abstract void removeWithTTL(byte[] key) throws PDException; /** - * 前缀查询 + * Prefix queries * * @param prefix * @return @@ -66,13 +65,12 @@ public abstract void putWithTTL(byte[] key, public abstract List scanPrefix(byte[] prefix) throws PDException; /** - * 前缀查询 + * Prefix queries * * @param prefix * @return * @throws PDException */ - public abstract List scanPrefix(Parser parser, byte[] prefix) throws PDException; public abstract List scanRange(byte[] start, byte[] end) throws PDException; @@ -81,13 +79,12 @@ public abstract List scanRange(Parser parser, byte[] start, byte[] end PDException; /** - * 检查Key是否存在 + * Check if the key exists * * @param key * @return * @throws PDException */ - public abstract boolean containsKey(byte[] key) throws PDException; public abstract long remove(byte[] key) throws PDException; diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/PartitionMeta.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/PartitionMeta.java index 713a0046d7..2952a1725d 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/PartitionMeta.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/PartitionMeta.java @@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 分区信息管理 + * Partition information management */ @Slf4j public class PartitionMeta extends MetadataRocksDBStore { @@ -46,7 +46,7 @@ public PartitionMeta(PDConfig pdConfig) { } /** - * 初始化，加载所有的分区 + * Initialize, load all partitions */ public void init() throws PDException { loadShardGroups(); @@ -69,7 +69,8 @@ private void loadGraphs() throws PDException { } /** - * partition 和 shard group分开存储，再init的时候，需要加载进来 + * The partition and shard group are stored separately, and when they are init, they need to + * be loaded * * @throws PDException */ @@ -89,7 +90,7 @@ private void loadPartitions(Metapb.Graph graph) throws PDException { } /** - * 根据id查找分区 (先从缓存找，再到数据库中找） + * Find partitions by ID (first from the cache, then from the database) * * @param graphName * @param partId @@ -124,7 +125,7 @@ public List getPartitionById(int partId) throws PDException { } /** - * 根据code查找分区 + * Find partitions based on code */ public Metapb.Partition getPartitionByCode(String graphName, long code) throws PDException { var pair = cache.getPartitionByCode(graphName, code); @@ -144,14 +145,12 @@ public Metapb.Graph getAndCreateGraph(String graphName, int partitionCount) thro partitionCount = pdConfig.getPartition().getTotalCount(); } - // 管理图，只有一个分区 if (graphName.endsWith("/s") || graphName.endsWith("/m")) { partitionCount = 1; } Metapb.Graph graph = cache.getGraph(graphName); if (graph == null) { - // 保存图信息 graph = Metapb.Graph.newBuilder() .setGraphName(graphName) .setPartitionCount(partitionCount) @@ -163,7 +162,7 @@ public Metapb.Graph getAndCreateGraph(String graphName, int partitionCount) thro } /** - * 保存分区信息 + * Save the partition information * * @param partition * @return @@ -179,14 +178,6 @@ public Metapb.Partition updatePartition(Metapb.Partition partition) throws PDExc return partition; } - /** - * 检查数据库，是否存在对应的图，不存在，则创建。 - * 更新partition的 version, conf version 和 shard list - * - * @param partition - * @return - * @throws PDException - */ public Metapb.Partition updateShardList(Metapb.Partition partition) throws PDException { if (!cache.hasGraph(partition.getGraphName())) { getAndCreateGraph(partition.getGraphName()); @@ -205,7 +196,7 @@ public Metapb.Partition updateShardList(Metapb.Partition partition) throws PDExc } /** - * 删除所有分区 + * Delete all partitions */ public long removeAllPartitions(String graphName) throws PDException { cache.removeAll(graphName); @@ -227,7 +218,7 @@ public void updatePartitionStats(Metapb.PartitionStats stats) throws PDException } /** - * 获取分区状态 + * Get the partition status */ public Metapb.PartitionStats getPartitionStats(String graphName, int id) throws PDException { byte[] prefix = MetadataKeyHelper.getPartitionStatusKey(graphName, id); @@ -235,7 +226,7 @@ public Metapb.PartitionStats getPartitionStats(String graphName, int id) throws } /** - * 获取分区状态 + * Get the partition status */ public List getPartitionStats(String graphName) throws PDException { byte[] prefix = MetadataKeyHelper.getPartitionStatusPrefixKey(graphName); @@ -243,7 +234,7 @@ public List getPartitionStats(String graphName) throws PD } /** - * 更新图信息 + * Update the diagram information * * @param graph * @return @@ -251,7 +242,6 @@ public List getPartitionStats(String graphName) throws PD public Metapb.Graph updateGraph(Metapb.Graph graph) throws PDException { log.info("updateGraph {}", graph); byte[] key = MetadataKeyHelper.getGraphKey(graph.getGraphName()); - // 保存图信息 put(key, graph.toByteString().toByteArray()); cache.updateGraph(graph); return graph; @@ -281,7 +271,7 @@ public Metapb.Graph getGraph(String graphName) throws PDException { } /** - * 删除图，并删除图id + * Delete the diagram and delete the diagram ID */ public long removeGraph(String graphName) throws PDException { byte[] key = MetadataKeyHelper.getGraphKey(graphName); diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/StoreInfoMeta.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/StoreInfoMeta.java index 2a50b0448c..3019578ede 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/StoreInfoMeta.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/StoreInfoMeta.java @@ -28,7 +28,7 @@ import lombok.extern.slf4j.Slf4j; /** - * Store信息存储 + * Store information storage */ @Slf4j public class StoreInfoMeta extends MetadataRocksDBStore { @@ -55,7 +55,7 @@ public static boolean shardGroupEquals(List g1, List } /** - * 更新Store信息 + * Update the Store information * * @param store * @throws PDException @@ -66,7 +66,7 @@ public void updateStore(Metapb.Store store) throws PDException { } /** - * 更新Store的存活状态 + * Update the survivability status of the store * * @param store */ @@ -87,7 +87,7 @@ public Metapb.Store getStore(Long storeId) throws PDException { } /** - * 获取所有的store + * Get all the stores * * @param graphName * @return @@ -99,9 +99,8 @@ public List getStores(String graphName) throws PDException { } /** - * 获取活跃的Store + * Get an active store * - * @param graphName * @return * @throws PDException */ @@ -120,7 +119,7 @@ public List getActiveStores() throws PDException { } /** - * 检查storeid是否存在 + * Check whether the storeID exists * * @param storeId * @return @@ -131,7 +130,7 @@ public boolean storeExists(Long storeId) throws PDException { } /** - * 更新存储状态信息 + * Update the storage status information * * @param storeStats */ @@ -185,7 +184,7 @@ public Metapb.StoreStats getStoreStats(long storeId) throws PDException { } /** - * @return store及状态信息 + * @return store and status information * @throws PDException */ public List getStoreStatus(boolean isActive) throws PDException { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/TaskInfoMeta.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/TaskInfoMeta.java index 756be71e98..5dbda2b097 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/TaskInfoMeta.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/TaskInfoMeta.java @@ -27,7 +27,7 @@ import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; /** - * 任务管理 + * Task management */ public class TaskInfoMeta extends MetadataRocksDBStore { @@ -36,7 +36,7 @@ public TaskInfoMeta(PDConfig pdConfig) { } /** - * 添加分区分裂任务 + * Add a partition splitting task */ public void addSplitTask(int groupID, Metapb.Partition partition, SplitPartition splitPartition) throws PDException { @@ -115,9 +115,9 @@ public List scanMoveTask(String graphName) throws PDException { } /** - * 按照prefix删除迁移任务，一次分组的 + * Delete the migration task by prefixing it and group them all at once * - * @param graphName 图名称 + * @param graphName graphName * @throws PDException io error */ public void removeMoveTaskPrefix(String graphName) throws PDException { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVOperation.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVOperation.java index 9169a248dc..b27252fa10 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVOperation.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVOperation.java @@ -55,7 +55,9 @@ public class KVOperation { private byte[] key; private byte[] value; - private Object attach; // 原始对象，用于本机处理，减少一次反序列化操作 + // Raw object, used for native processing, reducing the number of deserialization + // operations + private Object attach; private Object arg; private byte op; diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java index 9ed62b0e61..67734d1456 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java @@ -92,25 +92,25 @@ public boolean init(PDConfig.Raft config) { log.error("The RaftEngine parameter is incorrect." + " When RAFT is enabled, the number of peers " + "cannot be less than 3"); } - // 设置 Node 参数，包括日志存储路径和状态机实例 + // Set node parameters, including the log storage path and state machine instance NodeOptions nodeOptions = new NodeOptions(); nodeOptions.setFsm(stateMachine); nodeOptions.setEnableMetrics(true); - // 日志路径 + // Log path nodeOptions.setLogUri(raftPath + "/log"); - // raft 元数据路径 + // raft metadata path nodeOptions.setRaftMetaUri(raftPath + "/meta"); - // 快照路径 + // Snapshot path nodeOptions.setSnapshotUri(raftPath + "/snapshot"); - // 初始集群 + // Initial cluster nodeOptions.setInitialConf(initConf); - // 快照时间间隔 + // Snapshot interval nodeOptions.setSnapshotIntervalSecs(config.getSnapshotInterval()); nodeOptions.setRpcConnectTimeoutMs(config.getRpcTimeout()); nodeOptions.setRpcDefaultTimeout(config.getRpcTimeout()); nodeOptions.setRpcInstallSnapshotTimeout(config.getRpcTimeout()); - // 设置 raft 配置 + // Set the raft configuration RaftOptions raftOptions = nodeOptions.getRaftOptions(); nodeOptions.setEnableMetrics(true); @@ -118,7 +118,7 @@ public boolean init(PDConfig.Raft config) { final PeerId serverId = JRaftUtils.getPeerId(config.getAddress()); rpcServer = createRaftRpcServer(config.getAddress()); - // 构建 raft 组并启动 raft + // construct raft group and start raft this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOptions, rpcServer, true); this.raftNode = raftGroupService.start(false); @@ -128,7 +128,7 @@ public boolean init(PDConfig.Raft config) { } /** - * 创建 raft rpc server，用于 pd 之间通讯 + * Create a Raft RPC Server for communication between PDs */ private RpcServer createRaftRpcServer(String raftAddr) { Endpoint endpoint = JRaftUtils.getEndPoint(raftAddr); @@ -164,7 +164,7 @@ public boolean isLeader() { } /** - * 添加 Raft 任务，grpc 通过该接口给 raft 发送数据 + * Add a raft task, and grpc sends data to raft through this interface */ public void addTask(Task task) { if (!isLeader()) { @@ -193,7 +193,7 @@ public PeerId getLeader() { } /** - * 向 leader 发消息，获取 grpc 地址； + * Send a message to the leader to get the grpc address; */ public String getLeaderGrpcAddress() throws ExecutionException, InterruptedException { if (isLeader()) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcClient.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcClient.java index 6e47ce4e59..8c7398a53a 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcClient.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcClient.java @@ -46,7 +46,7 @@ public synchronized boolean init(final RpcOptions rpcOptions) { } /** - * 请求快照 + * Request a snapshot */ public CompletableFuture getGrpcAddress(final String address) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateMachine.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateMachine.java index e747518668..ec773ac6f1 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateMachine.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateMachine.java @@ -237,7 +237,6 @@ public boolean onSnapshotLoad(final SnapshotReader reader) { try { // TODO: remove file from meta - // SnapshotReader 沒有提供刪除文件的接口 FileUtils.deleteDirectory(new File(snapshotDir)); File file = new File(snapshotArchive); if (file.exists()) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftTaskHandler.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftTaskHandler.java index ec8120cc83..9bd4528a51 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftTaskHandler.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftTaskHandler.java @@ -20,7 +20,7 @@ import org.apache.hugegraph.pd.common.PDException; /** - * 接收raft发送的数据 + * Receives data sent by raft */ public interface RaftTaskHandler { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/RaftKVStore.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/RaftKVStore.java index ed97d13f71..b61f07ac1d 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/RaftKVStore.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/RaftKVStore.java @@ -79,7 +79,7 @@ public void put(byte[] key, byte[] value) throws PDException { } /** - * 查询可以不走raft，直接读取 + * Queries can be read without rafting */ @Override public byte[] get(byte[] key) throws PDException { @@ -180,7 +180,7 @@ public void close() { } /** - * 需要走Raft的真实操作 + * Need to walk the real operation of Raft */ private void doPut(byte[] key, byte[] value) throws PDException { diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index bccdeca4c8..e673e97d7c 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -37,43 +37,44 @@ license: license-path: ./conf/hugegraph.license grpc: port: 8686 - # grpc 的服务地址，部署时需要改为本地实际 IPv4 地址 + # The service address of grpc needs to be changed to the actual local IPv4 address when deploying. host: 127.0.0.1 server: - # rest 服务端口号 + # REST service port number port: 8620 pd: - # 存储路径 + # Storage path data-path: ./pd_data - # 自动扩容的检查周期，定时检查每个 store 的分区数量，自动进行分区数量平衡 + # The check cycle of automatic expansion regularly checks the number of partitions in each store and automatically balances the number of partitions patrol-interval: 1800 - # 初始 store 列表，在列表内的 store 自动激活 + # The initial store list, grpc IP: grpc port, the store in the list is automatically activated initial-store-count: 1 # grpc IP:grpc port initial-store-list: 127.0.0.1:8500 raft: - # 本机 raft 服务地址 + # The address of the local raft service address: 127.0.0.1:8610 - # pd 集群服务地址 + # The service address of the PD cluster peers-list: 127.0.0.1:8610 store: - # store 下线时间。超过该时间，认为 store 永久不可用，分配副本到其他机器，单位秒 + # The time when the store went offline. After that time, the store is considered permanently unavailable, and the replica is allocated to another machine, in seconds max-down-time: 172800 - # 是否开启 store 监控数据存储 + # Specifies whether to enable store monitoring data storage monitor_data_enabled: true - # 监控数据的间隔，minute (默认), hour, second + # The interval between monitoring data, minute, hour, second # default: 1 min * 1 day = 1440 monitor_data_interval: 1 minute - # 监控数据的保留时间 1 天; day, month, year + # Retention time of monitoring data is 1 day; day, month, year monitor_data_retention: 1 day initial-store-count: 1 partition: - # 默认每个分区副本数 + # Default number of replicas per partition default-shard-count: 1 - # 默认每机器最大副本数，初始分区数 = store-max-shard-count * store-number / default-shard-count + # The default maximum number of replicas per machine + # the initial number of partitions= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 12 diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template index 43f52df609..8b8f0d63c5 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template @@ -33,8 +33,8 @@ management: grpc: port: $GRPC_PORT$ - # grpc的服务地址, - #注意：部署时需要改为本地实际IPv4地址。 + # grpc's service address, + # Note: You need to change to the local actual Iv 4 address when deploying. host: $GRPC_HOST$ netty-server: max-inbound-message-size: 100MB @@ -43,29 +43,30 @@ server: port : $SERVER_PORT$ pd: - # 集群ID，区分不同的PD集群 + # Cluster ID: to distinguish different PD clusters patrol-interval: 2147483647 data-path: $PD_DATA_PATH$ raft: address: $RAFT_ADDRESS$ - # raft集群 + # raft cluster peers-list: $RAFT_PEERS_LIST$ - # 快照生成时间间隔，单位秒 + # The interval between snapshot generation, in seconds snapshotInterval: 300 metrics: true store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 + # If the store heartbeat timeout period exceeds this time, the store is temporarily unavailable and the leader is transferred to another replica in seconds keepAlive-timeout: 60 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 + # The time when the store went offline. After that time, the store is considered permanently unavailable, and the replica is allocated to another machine, in seconds max-down-time: 1800 partition: - # 默认分区总数 + # The default total number of partitions default-total-count: 30 - # 默认每个分区副本数 + # Default number of replicas per partition default-shard-count: 3 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 + # After the client registers, the maximum number of heartbeats is not reached, and after that, the + previous registration information will be deleted heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-grpc/pom.xml b/hugegraph-pd/hg-pd-grpc/pom.xml index cef49e957d..7df8622e19 100644 --- a/hugegraph-pd/hg-pd-grpc/pom.xml +++ b/hugegraph-pd/hg-pd-grpc/pom.xml @@ -92,23 +92,15 @@ io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} - ${project.basedir}/src/main/proto - - ${project.basedir}/src/main/java - false - - generate-sources - compile - compile-custom diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto index c4bb8bde10..65ab26a688 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto @@ -30,7 +30,6 @@ enum TaskType { Change_KeyRange = 5; } -// 一条任务信息 message Task { uint64 id = 1; TaskType type = 2; @@ -38,7 +37,6 @@ message Task { int64 start_timestamp = 4; metapb.Partition partition = 5; string message = 6; - //每个shard执行的任务状态 repeated ShardTaskState shardState = 7; ChangeShard changeShard = 9; SplitPartition splitPartition = 10; @@ -49,10 +47,10 @@ message Task { enum TaskState{ Task_Unknown = 0; - Task_Ready = 1; //任务就绪 - Task_Doing = 2; //执行中 - Task_Done = 3; //完成 - Task_Exit = 4; //退出 + Task_Ready = 1; + Task_Doing = 2; + Task_Done = 3; + Task_Exit = 4; Task_Stop = 10; Task_Success = 11; Task_Failure = 12; diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto index a8a695be04..2d361de662 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto @@ -21,17 +21,17 @@ option java_package = "org.apache.hugegraph.pd.grpc"; import "google/protobuf/any.proto"; enum ClusterState{ - // 集群健康 + // Cluster health Cluster_OK = 0; - // 分区警告，存在部分故障节点，短时间不影响读写 + // Partition warning: There are some faulty nodes, which do not affect read/write for a short time Cluster_Warn = 2; - // 分区下线，可以读，无法写 + // The partition is offline, which can be read but cannot be written Cluster_Offline = 10; - // 分区故障，无法读写,需要尽快修复故障节点。 + // If the partition is faulty and cannot be read or written, you need to repair the faulty node as soon as possible. Cluster_Fault = 11; Cluster_Not_Ready = -1; } -// 集群状态 +// Cluster status message ClusterStats{ ClusterState state = 1; string message = 2; @@ -40,15 +40,13 @@ message ClusterStats{ enum StoreState { Unknown = 0; - // 未激活 + // Not activated Pending = 4; - // 在线 + // online Up = 1; - // 离线 + // Offline Offline = 2; - // 下线中 Exiting = 5; - // 已下线 Tombstone = 3; } @@ -64,7 +62,7 @@ message Store { string address = 2; string raft_address = 3; repeated StoreLabel labels = 4; - // Store软件版本号 + // Store Software version number string version = 5; StoreState state = 6; // The start timestamp of the current store @@ -73,7 +71,7 @@ message Store { // The last heartbeat timestamp of the store. int64 last_heartbeat = 9; StoreStats stats = 10; - // 数据格式版本号 + // The version number of the data format int32 data_version = 11; int32 cores = 12; string data_path = 13; @@ -103,38 +101,37 @@ message ShardGroup{ message Graph { string graph_name = 2; - // 分区数量，0表示无效，不能大于raft分组总数 + // The number of partitions, 0 indicates invalid and cannot be greater than the total number of raft packets int32 partition_count = 3; - // 当前工作状态 PartitionState state = 10; string message = 11; GraphState graph_state = 12; } -// 分区工作状态 +// Partition working status enum PartitionState{ PState_None = 0; // PState_Normal = 1; - // 分区警告，存在部分故障节点，短时间不影响读写 + // Partition warning: There are some faulty nodes, which do not affect read/write for a short time PState_Warn = 2; - // 分区下线，可以读，无法写 + // The partition is offline, which can be read but cannot be written PState_Offline = 10; - // 分区故障，无法读写,需要尽快修复故障节点。 + // If the partition is faulty and cannot be read or written, you need to repair the faulty node as soon as possible. PState_Fault = 11; } message PartitionV36 { uint32 id = 1; string graph_name = 3; - // 分区范围 [start_key, end_key). + // Partition range [start_key, end_key]. uint64 start_key = 4; uint64 end_key = 5; repeated Shard shards = 6; - // Leader任期，leader切换后递增 + // Leader term, leader increment after switching uint64 version = 7; - // shards版本号，每次改变后递增 + // shards The version number, which is incremented with each change uint64 conf_ver = 8; - // 当前工作状态 + // Current working status PartitionState state = 10; string message = 11; } @@ -142,16 +139,16 @@ message PartitionV36 { message Partition { uint32 id = 1; string graph_name = 3; - // 分区范围 [start_key, end_key). + // Partition range [start_key, end_key]. uint64 start_key = 4; uint64 end_key = 5; - // Partition 对象不在保存 shard list（根据对应的shard group 去查询), version 和 conf version不再有实际的意义 + // The partition object no longer stores the shard list (which is queried according to the corresponding shard group), and the version and conf version are no longer meaningful // repeated Shard shards = 6; - // key range 每次改变后递增 + // key range increment after each change uint64 version = 7; - // shards版本号，每次改变后递增 + // shards The version number, which is incremented with each change // uint64 conf_ver = 8; - // 当前工作状态 + // Current working status PartitionState state = 10; string message = 11; } @@ -159,21 +156,21 @@ message Partition { message PartitionShard { metapb.Partition partition = 1; metapb.Shard leader = 2; - // 离线的Shard + // Offline Shard repeated metapb.Shard offline_shards = 3; } -// 记录分区所在的存储位置 +// The storage location where the record partition is located message PartitionStore { uint32 partition_id = 1; string graph_name = 3; - // 存储位置 + // Storage location string store_location = 4; } message PartitionRaft { uint32 partition_id = 1; string graph_name = 3; - // 存储位置 + // Storage location string raft_location = 4; } @@ -181,42 +178,43 @@ message ShardStats{ uint64 store_id = 2; ShardRole role = 3; ShardState state = 4; - // 安装快照的进度 + // The progress of the installation of the snapshot uint32 progress = 5; } + message PartitionStats{ uint32 id = 1; - // raft分组的任期. + // raft Term of office of the group. uint64 leader_term = 2; repeated string graph_name = 3; metapb.Shard leader = 4; - // 离线 shards + // Offline shards repeated metapb.Shard shard = 5; repeated metapb.Shard learner = 6; uint64 conf_ver = 7; - // 分区状态 + // partition status PartitionState state = 8; repeated ShardStats shardStats = 9; - // 分区近似大小 + // The approximate size of the partition uint64 approximate_size = 10; - // 分区key的近似数量 + // The approximate number of partition keys uint64 approximate_keys = 13; // heartbeat timestamp int64 timestamp = 16; } message GraphStats{ - // 图名 + // graph string graph_name = 1; - // 分区近似大小 + // The approximate size of the partition uint64 approximate_size = 2; - // 分区key的近似数量 + // The approximate number of partition keys uint64 approximate_keys = 3; // // committed index // uint64 committed_index = 4; uint32 partition_id = 5; ShardRole role = 6; - // 当前工作状态 + // Current working status PartitionState work_state = 8; } @@ -252,11 +250,11 @@ message QueryStats { enum ShardState{ SState_None = 0; - // 正常 + // Normal SState_Normal = 1; - // 安装快照 + // Install snapshots SState_Snapshot = 2; - // 离线 + // Offline SState_Offline = 10; } @@ -310,14 +308,14 @@ message StoreStats { repeated RecordPair system_metrics = 25; } -// 分区查询条件 +// Partition query criteria message PartitionQuery{ - optional uint64 store_id = 1; // 0 表示查询条件不包含store_id + optional uint64 store_id = 1; // 0 Indicates that the query criteria do not contain store_id optional string graph_name = 2; optional uint32 partition_id = 4; } -//PD 节点信息 +// PD Node information message Member { uint64 cluster_id = 1; string raft_url = 3; @@ -329,37 +327,38 @@ message Member { string replicator_state = 9; } -// 图空间配置 +// Graph space configuration message GraphSpace{ string name = 1; - // 最大占用存储 + // Maximum occupied storage uint64 storage_limit = 2; - // 已使用空间 + // Space used uint64 used_size = 3; - // 修改时间 + // Modify time uint64 timestamp = 10; } -// PD 配置 +// PD Config message PDConfig{ uint64 version = 1; - // 分区数量, 初始化根据Store数量动态计算，分裂后进行修改 + // The number of partitions is dynamically calculated based on the number of stores at the initialization and modified after splitting + // The exact count needs to be determined based on the store group int32 partition_count = 2; - // 每分区副本数量 + // Number of replicas per partition int32 shard_count = 3; - // pd集群列表 + // pd List of clusters string peers_list = 4; - // 集群中最少store数量 + // The minimum number of stores in the cluster int32 min_store_count = 6; - // 每个store最大副本数 + // Maximum number of replicas per store int32 max_Shards_Per_Store = 7; - // 修改时间 + // Modify time uint64 timestamp = 10; } -//消息持久化 +// Message persistence message QueueItem{ string item_id = 1; string item_class = 2; @@ -387,8 +386,7 @@ enum GraphMode{ } enum GraphModeReason{ - Empty = 0; // 空 - Initiative = 1; // 主动的状态设置 - Quota = 2; // 达到限额条件 - + Empty = 0; + Initiative = 1; // Active status settings + Quota = 2; // The limit condition is reached } diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto index c9eec81494..c2b55c2787 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto @@ -22,9 +22,7 @@ option java_package = "org.apache.hugegraph.pd.grpc.common"; option java_outer_classname = "HgPdCommonProto"; message RequestHeader { - // 集群 ID. uint64 cluster_id = 1; - // 发送者 ID. uint64 sender_id = 2; } diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto index fb8940df6c..afb6d6287d 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto @@ -56,8 +56,8 @@ message PulseAckRequest { int64 notice_id = 2; } -// 分区心跳，分区的peer增减、leader改变等事件发生时，由leader发送心跳。 -// 同时pd对分区进行shard增减通过Response发送给leader +// When an event such as a partition heartbeat occurs such as the increase or decrease of peers in a partition or the change of leader, the leader sends a heartbeat. +// At the same time, the pd adds or decreases shards to the partition and sends the response to the leader message PartitionHeartbeatRequest { RequestHeader header = 1; // Leader Peer sending the heartbeat @@ -83,15 +83,15 @@ message PartitionHeartbeatResponse { ChangeShard change_shard = 4; TransferLeader transfer_leader = 5; - // 拆分成多个分区，第一个SplitPartition是原分区，从第二开始是新分区 + // Split into multiple partitions, with the first SplitPartition being the original partition and the second starting being the new partition SplitPartition split_partition = 6; - // rocksdb compaction 指定的表，null是针对所有 + // rocksdb compaction specifies the table, null is for all DbCompaction db_compaction = 7; - // 将partition的数据，迁移到 target + // Migrate data from the partition to the target MovePartition move_partition = 8; - // 清理partition的graph的数据 + // Clean up the data for the partition of the graph CleanPartition clean_partition = 9; - // partition key range 变化 + // partition key range variation PartitionKeyRange key_range = 10; } @@ -113,11 +113,11 @@ message DbCompaction { string table_name = 3; } -message MovePartition{ - // target partition的key range为，迁移后的新range +message MovePartition { + // The new range after migration metapb.Partition target_partition = 1; - // partition 的 key start 和 key end的所有数据， - // 会迁移到 target partition 上 + // partition's key start and key end, + // will migrate to target partition uint64 key_start = 2; uint64 key_end = 3; } @@ -126,7 +126,7 @@ message CleanPartition { uint64 key_start = 1; uint64 key_end = 2; CleanType clean_type = 3; - bool delete_partition = 4; //是否删除分区 + bool delete_partition = 4; // Whether to delete the partition } message PartitionKeyRange{ @@ -159,12 +159,12 @@ enum ConfChangeType { CONF_CHANGE_TYPE_ADD_NODE = 1; CONF_CHANGE_TYPE_REMOVE_NODE = 2; CONF_CHANGE_TYPE_ADD_LEARNER_NODE = 3; - CONF_CHANGE_TYPE_ADJUST = 4; // 调整shard，leader根据新的配置动态增减。 + CONF_CHANGE_TYPE_ADJUST = 4; // Adjust the shard, and the leader dynamically increases or decreases according to the new configuration. } enum CleanType { - CLEAN_TYPE_KEEP_RANGE = 0; // 仅保留这个range - CLEAN_TYPE_EXCLUDE_RANGE = 1; // 删除这个range + CLEAN_TYPE_KEEP_RANGE = 0; // Only this range remains + CLEAN_TYPE_EXCLUDE_RANGE = 1; // Delete this range } enum PdInstructionType { diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto index febc41f522..6d0c016c2c 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto @@ -98,6 +98,5 @@ enum NodeEventType { NODE_EVENT_TYPE_NODE_ONLINE = 1; NODE_EVENT_TYPE_NODE_OFFLINE = 2; NODE_EVENT_TYPE_NODE_RAFT_CHANGE = 3; - // pd leader 变更 NODE_EVENT_TYPE_PD_LEADER_CHANGE = 4; } diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto index 4e293ca08e..f7754824ec 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto @@ -24,83 +24,83 @@ import "metaTask.proto"; option java_package = "org.apache.hugegraph.pd.grpc"; service PD { - // 注册store，首次注册会生成新的store_id， store_id是store唯一标识 + // Register store, the first registration will generate a new store_id, store_id is the unique identifier of the store rpc RegisterStore(RegisterStoreRequest) returns (RegisterStoreResponse) {} rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} - // 修改Store状态等信息. + // Modify Store status and other information. rpc SetStore(SetStoreRequest) returns (SetStoreResponse) {} - // 根据可以查找所属分区 + // Look up the partition based on the key rpc DelStore(DetStoreRequest) returns (DetStoreResponse) {} rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} - // 根据可以查找所属分区 + // Look up the partition based on the key rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse) {} - // 根据HashCode查找所属分区 + // Look up the partition based on HashCode rpc GetPartitionByCode(GetPartitionByCodeRequest) returns (GetPartitionResponse) {} - // 根据PartitionID返回分区 + // Return partition by PartitionID rpc GetPartitionByID(GetPartitionByIDRequest) returns (GetPartitionResponse) {} rpc ScanPartitions(ScanPartitionsRequest) returns (ScanPartitionsResponse) {} - // 更新分区信息，主要用来更新分区key范围，调用此接口需谨慎，否则会造成数据丢失。 + // Update partition information, mainly used to update partition key range, call this interface carefully, otherwise data loss will occur. rpc UpdatePartition(UpdatePartitionRequest) returns (UpdatePartitionResponse) {} - // 根据可以查找所属分区 + // Look up the partition based on the key rpc DelPartition(DelPartitionRequest) returns (DelPartitionResponse) {} - // 根据条件查询分区信息, 包括Store、Graph等条件 + // Query partition information based on conditions, including Store, Graph and other conditions rpc QueryPartitions(QueryPartitionsRequest) returns (QueryPartitionsResponse){} - // 读取图信息 + // Read graph information rpc GetGraph(GetGraphRequest) returns (GetGraphResponse){} - // 修改图信息 + // Modify graph information rpc SetGraph(SetGraphRequest) returns (SetGraphResponse){} rpc DelGraph(DelGraphRequest) returns (DelGraphResponse){} - // 全局唯一自增ID + // Global unique incremental ID rpc GetId(GetIdRequest) returns (GetIdResponse){} rpc ResetId(ResetIdRequest) returns (ResetIdResponse){} - // PD的集群列表 + // PD cluster list rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} rpc GetStoreStatus(GetAllStoresRequest) returns (GetAllStoresResponse) {} rpc GetPDConfig(GetPDConfigRequest) returns (GetPDConfigResponse){} rpc SetPDConfig(SetPDConfigRequest) returns (SetPDConfigResponse){} rpc GetGraphSpace(GetGraphSpaceRequest) returns (GetGraphSpaceResponse){} rpc SetGraphSpace(SetGraphSpaceRequest) returns (SetGraphSpaceResponse){} - // 获取集群健康状态 + // Get cluster health status rpc GetClusterStats(GetClusterStatsRequest) returns (GetClusterStatsResponse){} - // 替换PD的集群节点 + // Replace PD cluster nodes rpc ChangePeerList(ChangePeerListRequest) returns (getChangePeerListResponse) {} - // 数据分裂 + // Data splitting rpc SplitData(SplitDataRequest) returns (SplitDataResponse){} rpc SplitGraphData(SplitGraphDataRequest) returns (SplitDataResponse) {} - // 数据迁移 + // Data migration rpc MovePartition(MovePartitionRequest) returns (MovePartitionResponse){} - // 汇报分区分裂等任务执行结果 + // Report partition splitting and other task execution results rpc ReportTask(ReportTaskRequest) returns (ReportTaskResponse){} rpc GetPartitionStats(GetPartitionStatsRequest) returns (GetPartitionStatsResponse){} - //平衡store中分区leader的数量 + // Balance the number of partition leaders in the store rpc BalanceLeaders(BalanceLeadersRequest) returns (BalanceLeadersResponse){} - // 替换license文件 + // Replace license file rpc PutLicense(PutLicenseRequest) returns (PutLicenseResponse){} - // 通知rocksdb进行compaction + // Notify rocksdb to perform compaction rpc DbCompaction(DbCompactionRequest) returns (DbCompactionResponse){} - // 合并分区 + // Merge partitions rpc CombineCluster(CombineClusterRequest) returns (CombineClusterResponse){} - // 单个图缩容 + // Shrink a single graph rpc CombineGraph(CombineGraphRequest) returns (CombineGraphResponse) {} // shard group rpc GetShardGroup(GetShardGroupRequest) returns (GetShardGroupResponse){} rpc UpdateShardGroup(UpdateShardGroupRequest) returns (UpdateShardGroupResponse){} - // 删除掉shard group + // Delete shard group rpc DeleteShardGroup(DeleteShardGroupRequest) returns (DeleteShardGroupResponse) {} - // shard group 运维相关的处理 + // shard group operation related to operation rpc UpdateShardGroupOp(ChangeShardRequest) returns (ChangeShardResponse){} - // change shard + // Change shard rpc ChangeShard(ChangeShardRequest) returns (ChangeShardResponse) {} - // 更新pd raft + // Update pd raft rpc updatePdRaft(UpdatePdRaftRequest) returns (UpdatePdRaftResponse) {} rpc getCache(GetGraphRequest) returns (CacheResponse) {} @@ -108,9 +108,7 @@ service PD { } message RequestHeader { - // 集群 ID. uint64 cluster_id = 1; - // 发送者 ID. uint64 sender_id = 2; } @@ -141,21 +139,21 @@ enum ErrorType { ROCKSDB_SAVE_SNAPSHOT_ERROR = 1005; ROCKSDB_LOAD_SNAPSHOT_ERROR = 1006; - // 当前集群状态禁止分裂 + // Current cluster state prohibits splitting Cluster_State_Forbid_Splitting = 1007; - // 正在分裂中 + // Currently splitting Split_Partition_Doing = 1008; - // store上分区数量超过上限 + // The number of partitions on the store exceeds the limit Too_Many_Partitions_Per_Store = 1009; - // license 错误 + // License error LICENSE_ERROR = 107; - // license 认证错误 + // License authentication error LICENSE_VERIFY_ERROR = 108; - //分区下线正在进行 + // Partition offline is in progress Store_Tombstone_Doing = 1010; - // 不合法的分裂个数 + // Invalid number of splits Invalid_Split_Partition_Count = 1011; } @@ -193,7 +191,7 @@ message RegisterStoreRequest { message RegisterStoreResponse { ResponseHeader header = 1; - // 初次注册，返回新的store_id + // Upon initial registration, returns the new store_id uint64 store_id = 2; } @@ -204,16 +202,15 @@ message SetStoreRequest { message SetStoreResponse { ResponseHeader header = 1; - // 返回修改后的Store + // Returns the modified Store metapb.Store store = 2; } - -// 返回graph_name所在的所有store，如果graph_name为空值，则返回系统所有的store +// Returns all stores where graph_name is located. If graph_name is empty, returns all system stores. message GetAllStoresRequest { RequestHeader header = 1; string graph_name = 2; - // 是否返回离线的store + // Whether to exclude offline stores bool exclude_offline_stores = 3; } @@ -254,7 +251,7 @@ message GetPartitionResponse { ResponseHeader header = 1; metapb.Partition partition = 2; metapb.Shard leader = 3; - // 离线的Shard + // Offline Shards repeated metapb.Shard offline_shards = 4; } @@ -435,65 +432,65 @@ enum OperationMode { Expert = 1; } -message SplitDataParam{ - // 被分裂的源分区ID +message SplitDataParam { + // ID of the source partition being split uint32 partition_id = 1; - //目标分区数量 + // Number of target partitions uint32 count = 2; } -message SplitDataRequest{ +message SplitDataRequest { RequestHeader header = 1; - //工作模式 - // Auto：自动分裂，每个Store上分区数达到最大值 - // Expert:专家模式，需要指定splitParams + // Operation mode + // Auto: Automatic splitting, when the number of partitions on each Store reaches the maximum value + // Expert: Expert mode, requires specifying splitParams OperationMode mode = 2; repeated SplitDataParam param = 3; } -message SplitGraphDataRequest{ +message SplitGraphDataRequest { RequestHeader header = 1; - //工作模式 + // Operation mode string graph_name = 2; uint32 to_count = 3; } -message SplitDataResponse{ +message SplitDataResponse { ResponseHeader header = 1; } -message MovePartitionParam{ +message MovePartitionParam { uint32 partition_id = 1; uint64 src_store_id = 2; uint64 dst_store_id = 3; } -message MovePartitionRequest{ +message MovePartitionRequest { RequestHeader header = 1; - //工作模式 - // Auto：自动转移，达到每个Store上分区数量相同 - // Expert:专家模式，需要指定transferParams + // Operation mode + // Auto: Automatic transfer, aiming for equal number of partitions on each Store + // Expert: Expert mode, requires specifying transferParams OperationMode mode = 2; repeated MovePartitionParam param = 3; } -message MovePartitionResponse{ +message MovePartitionResponse { ResponseHeader header = 1; } -message ReportTaskRequest{ +message ReportTaskRequest { RequestHeader header = 1; metaTask.Task task = 2; } -message ReportTaskResponse{ +message ReportTaskResponse { ResponseHeader header = 1; } -message GetPartitionStatsRequest{ +message GetPartitionStatsRequest { RequestHeader header = 1; uint32 partition_id = 2; - // 如果未空，返回所有图的同一分区ID + // If empty, returns the same partition ID for all graphs string graph_name = 4; } @@ -596,7 +593,7 @@ message UpdatePdRaftResponse{ } message CacheResponse { ResponseHeader header = 1; - // 返回修改后的Store + // Returns the modified Store repeated metapb.Store stores = 2; repeated metapb.ShardGroup shards = 3; repeated metapb.Graph graphs = 4; diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/boot/HugePDServer.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/boot/HugePDServer.java index 815005259a..cf105680db 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/boot/HugePDServer.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/boot/HugePDServer.java @@ -24,7 +24,7 @@ import com.alipay.remoting.util.StringUtils; /** - * PD 服务启动类 + * PD service startup class */ @ComponentScan(basePackages = {"org.apache.hugegraph.pd"}) @SpringBootApplication diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/pulse/PdInstructionSubject.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/pulse/PdInstructionSubject.java index e123384b2d..cf1b61553c 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/pulse/PdInstructionSubject.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/pulse/PdInstructionSubject.java @@ -38,7 +38,7 @@ String toNoticeString(PulseResponse res) { } /** - * pd单纯的向pulse发送的指令，不接收对应的notice + * Simply send a command to pulse, and do not receive the corresponding notice * * @return null */ diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/API.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/API.java index d748d23a71..a2287cb83e 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/API.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/API.java @@ -107,7 +107,7 @@ public String toJSON(Map> values) { } builder.append(","); }); - builder.deleteCharAt(builder.length() - 1); //删除最后一个逗号 + builder.deleteCharAt(builder.length() - 1); } builder.append("]").append(COMMA); } diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/GraphAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/GraphAPI.java index 68d80beb4b..0c25d78c38 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/GraphAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/GraphAPI.java @@ -143,7 +143,7 @@ public RestApiResponse getGraph(HttpServletRequest request) throws statistics = new GraphStatistics(graph); response.setData(statistics); } else { - response.setData(new HashMap()); //没有该图 + response.setData(new HashMap()); } response.setStatus(Pdpb.ErrorType.OK.getNumber()); response.setMessage(Pdpb.ErrorType.OK.name()); @@ -176,7 +176,7 @@ public Shard(Metapb.ShardStats shardStats, long partitionId) { public Shard(Metapb.Shard shard, long partitionId) { this.role = String.valueOf(shard.getRole()); this.storeId = shard.getStoreId(); - this.state = Metapb.ShardState.SState_Normal.name(); //gshard的状态默认为normal + this.state = Metapb.ShardState.SState_Normal.name(); this.progress = 0; this.partitionId = partitionId; } @@ -236,7 +236,7 @@ public Partition(Metapb.Partition pt, Metapb.PartitionStats partitionStats) { @Data class GraphStatistics { - //图统计信息 + // Graph statistics String graphName; long partitionCount; String state; @@ -255,7 +255,7 @@ public GraphStatistics(Metapb.Graph graph) throws PDException { graphName = graph.getGraphName(); partitionCount = graph.getPartitionCount(); state = String.valueOf(graph.getState()); - // 数据量及key的数量 + // The amount of data and the number of keys List stores = pdRestService.getStores(graphName); for (Metapb.Store store : stores) { List graphStatsList = store.getStats().getGraphStatsList(); @@ -272,7 +272,7 @@ public GraphStatistics(Metapb.Graph graph) throws PDException { List resultPartitionList = new ArrayList<>(); List tmpPartitions = pdRestService.getPartitions(graphName); if ((tmpPartitions != null) && (!tmpPartitions.isEmpty())) { - // 需要返回的分区信息 + // The partition information to be returned for (Metapb.Partition partition : tmpPartitions) { Metapb.PartitionStats partitionStats = pdRestService .getPartitionStats(graphName, partition.getId()); @@ -282,7 +282,7 @@ public GraphStatistics(Metapb.Graph graph) throws PDException { } } partitions = resultPartitionList; - // 隐去图名后面的 /g /m /s + // Hide /g /m /s after the title of the graph final int postfixLength = 2; graphName = graphName.substring(0, graphName.length() - postfixLength); } diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java index 89f6e86241..61f3c5a2c6 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java @@ -98,7 +98,6 @@ public RestApiResponse cluster() throws InterruptedException, ExecutionException statistics.onlineStoreSize = pdService.getStoreNodeService().getActiveStores().size(); statistics.offlineStoreSize = statistics.storeSize - statistics.onlineStoreSize; List graphs = pdRestService.getGraphs(); - // 图的数量，只统计/g statistics.graphSize = graphs.stream().filter((g) -> (g.getGraphName() != null) && (g.getGraphName().endsWith("/g"))) @@ -112,11 +111,15 @@ public RestApiResponse cluster() throws InterruptedException, ExecutionException statistics.dataSize += graphStats.getApproximateSize(); } } - // 数据状态：根据图的状态推出数据状态,枚举值越大，问题越严重， 默认为正常状态 + // Data status: The data status is deduced based on the state of the graph, the + // larger the enumeration value, the more serious the problem, and the default is the + // normal state Metapb.PartitionState dataState = Metapb.PartitionState.PState_Normal; for (Metapb.Graph graph : pdRestService.getGraphs()) { if (graph.getState() == Metapb.PartitionState.UNRECOGNIZED) { - continue; // 未识别不参与对比，不然会抛出异常 + // If it is not recognized, it will not participate in the + // comparison, otherwise an exception will be thrown + continue; } if ((graph.getState() != null) && (graph.getState().getNumber() > dataState.getNumber())) { @@ -174,9 +177,9 @@ class Member { String state; String dataPath; String role; - String serviceName; //服务名称，自定义属性 - String serviceVersion; //静态定义 - long startTimeStamp; //进程启动时间 + String serviceName; // Service name, custom attributes + String serviceVersion; // Static definitions + long startTimeStamp; // The time when the process started public Member(Metapb.Member member) { if (member != null) { @@ -200,31 +203,31 @@ public Member() { class Statistics { /** - * 集群状态 + * Cluster status, default of the cluster */ String state; /** - * 数据状态 + * Data status */ String dataState; /** - * pd集群成员 + * pd Cluster members */ List pdList; /** - * pd集群的leader + * pd The leader of the cluster */ Member pdLeader; /** - * pd集群的大小 + * pd The size of the cluster */ int memberSize; /** - * stores列表 + * stores list */ List stores; /** - * store的数量 + * store quantity */ int storeSize; /** @@ -232,27 +235,27 @@ class Statistics { */ int onlineStoreSize; /** - * 离线的store的数量 + * The number of stores that are offline */ int offlineStoreSize; /** - * 图的数量 + * The number of graphs */ long graphSize; /** - * 分区的数量 + * The number of partitions */ int partitionSize; /** - * 分区副本数 + * Number of partition replicas */ int shardCount; /** - * key的数量 + * The number of keys */ long keyCount; /** - * 数据量 + * Amount of data */ long dataSize; diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/MemberAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/MemberAPI.java index d0078b5db6..c6542c47ae 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/MemberAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/MemberAPI.java @@ -216,9 +216,9 @@ class Member { String dataPath; String role; String replicateState; - String serviceName; //服务名称，自定义属性 - String serviceVersion; //静态定义 - long startTimeStamp; //启动时间，暂时取进程的启动时间 + String serviceName; // Service name, custom attributes + String serviceVersion; // Static definitions + long startTimeStamp; // Startup time: temporarily takes the startup time of the process public Member(Metapb.Member member) { if (member != null) { @@ -236,6 +236,7 @@ public Member(Metapb.Member member) { } @Useless("delete later") - public Member() {} + public Member() { + } } } diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/PartitionAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/PartitionAPI.java index bdbdec39dc..14576a4f5e 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/PartitionAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/PartitionAPI.java @@ -58,12 +58,12 @@ public class PartitionAPI extends API { @GetMapping(value = "/highLevelPartitions", produces = MediaType.APPLICATION_JSON_VALUE) public RestApiResponse getHighLevelPartitions() { - // 分区下多个图的信息 + // Information about multiple graphs under the partition Map> partitions2GraphsMap = new HashMap<>(); Map resultPartitionsMap = new HashMap<>(); - // 每一个分区的keyCount， 只从leader处取出 + // The keyCount of each partition is only taken from the leader Map partition2KeyCount = new HashMap<>(); - // 每一个分区的dataSize， 只从leader处取出 + // The dataSize of each partition is only taken from the leader Map partition2DataSize = new HashMap<>(); List stores; Map storesMap = new HashMap<>(); @@ -77,20 +77,19 @@ public RestApiResponse getHighLevelPartitions() { storesMap.put(store.getId(), store); List graphStatsList = store.getStats().getGraphStatsList(); for (Metapb.GraphStats graphStats : graphStatsList) { - // 获取分区保存的图信息（只从leader处取出来） + // Obtaining Graph Information Saved by a Partition (Only from the Leader) if (Metapb.ShardRole.Leader != graphStats.getRole()) { continue; } - // 计算分区的keyCount（不区分图） + // Calculating the key count of partitions (indiscriminate graphs) partition2KeyCount.put(graphStats.getPartitionId(), partition2KeyCount.getOrDefault(graphStats.getPartitionId(), graphStats.getApproximateKeys())); - // 计算分区的dataSize, 通过累加图的大小实现 + // The dataSize of the partition is calculated by adding the size of the graph partition2DataSize.put(graphStats.getPartitionId(), partition2DataSize.getOrDefault(graphStats.getPartitionId(), 0L) + graphStats.getApproximateSize()); - // 构造分区下的图信息 if (partitions2GraphsMap.get(graphStats.getPartitionId()) == null) { partitions2GraphsMap.put(graphStats.getPartitionId(), new HashMap()); @@ -100,10 +99,10 @@ public RestApiResponse getHighLevelPartitions() { partitionGraphsMap.put(graphStats.getGraphName(), new GraphStats(graphStats)); } } - // 构造分区的所有需返回的信息 + // Construct all the information that needs to be returned for the partition List partitionList = pdRestService.getPartitions(""); for (Metapb.Partition partition : partitionList) { - // 补充分区内图信息的startKey, endKey + // Supplement the startKey and endKey of the partition image if (partitions2GraphsMap.get(partition.getId()) != null) { GraphStats graphStats = partitions2GraphsMap.get(partition.getId()).get(partition.getGraphName()); @@ -112,7 +111,7 @@ public RestApiResponse getHighLevelPartitions() { graphStats.endKey = partition.getEndKey(); } } - // 构造分区整体信息（不区分图） + // Construct the overall information of the partition (regardless of the diagram) if ((resultPartitionsMap.get(partition.getId()) == null) && (!partition.getGraphName().endsWith("/s")) ) { @@ -124,7 +123,7 @@ public RestApiResponse getHighLevelPartitions() { log.error("getPartitionStats error", e); partitionStats = null; } - // 初始化分区信息 + // Initialize the partition information HighLevelPartition resultPartition = new HighLevelPartition(partition, partitionStats); resultPartition.keyCount = @@ -132,28 +131,30 @@ public RestApiResponse getHighLevelPartitions() { resultPartition.dataSize = partition2DataSize.getOrDefault(resultPartition.partitionId, 0L); for (ShardStats shard : resultPartition.shards) { - // 对副本的地址，分区信息赋值 + // Assign values to the address and partition information of the replica shard.address = storesMap.get(shard.storeId).getAddress(); shard.partitionId = partition.getId(); } if ((partitionStats != null) && (partitionStats.getLeader() != null)) { - long storeId = partitionStats.getLeader().getStoreId(); // 获取leader的storeId + long storeId = partitionStats.getLeader().getStoreId(); resultPartition.leaderAddress = - storesMap.get(storeId).getAddress(); // 获取leader的address + storesMap.get(storeId).getAddress(); } resultPartitionsMap.put(partition.getId(), resultPartition); } } - // 构造需返回的分区下的图列表，只返回/g, 且按名称排序 + // Construct a list of graphs under the partitions to be returned, return only /g, and + // sort by name for (Map.Entry entry : resultPartitionsMap.entrySet()) { Integer partitionId = entry.getKey(); HighLevelPartition currentPartition = resultPartitionsMap.get(partitionId); Map graphsMap = partitions2GraphsMap - .getOrDefault(partitionId, new HashMap<>()); // 避免后面出现空指针异常 + .getOrDefault(partitionId, + new HashMap<>()); // Avoid null pointer exceptions at the back ArrayList graphsList = new ArrayList<>(); for (Map.Entry entry1 : graphsMap.entrySet()) { if (!entry1.getKey().endsWith("/g")) { - continue; // 只保留/g的图 + continue; // Only the graph of /g is kept } String graphName = entry1.getKey(); GraphStats tmpGraph = graphsMap.get(graphName); @@ -181,10 +182,10 @@ public RestApiResponse getHighLevelPartitions() { @GetMapping(value = "/partitions", produces = MediaType.APPLICATION_JSON_VALUE) public RestApiResponse getPartitions() { try { - List partitions = new ArrayList<>();//需返回的分区对象 + List partitions = new ArrayList<>(); List partitionList = pdRestService.getPartitions(""); List stores = pdRestService.getStoreStats(false); - //分区的raftNode的状态 + // The status of the raft node of the partition HashMap> raftMap = new HashMap<>(); HashMap> shardIndexMap = new HashMap<>(); @@ -392,7 +393,7 @@ class HighLevelPartition { long dataSize; String shardState; int progress; - long raftTerm; //任期 + long raftTerm; List graphs; List shards; String failureCause = ""; @@ -424,7 +425,7 @@ class HighLevelPartition { log.error("get shard list failed, {}", e.getMessage()); } } - // 综合所有副本的状态，给shardState赋值 + // Synthesize the state of all replicas and assign a value to shardState shardState = tmpShardState.name(); } } @@ -456,7 +457,7 @@ class ShardStats { String role; String state; int progress; - //额外属性 + // Extra attributes long partitionId; String address; @@ -468,7 +469,7 @@ class ShardStats { } ShardStats(Metapb.Shard shard) { - //当没有shardStats的初始化方法 + // When there is no initialization method for shardStats storeId = shard.getStoreId(); role = String.valueOf(shard.getRole()); state = Metapb.ShardState.SState_Normal.name(); diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/ShardAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/ShardAPI.java index 6cb5b09da3..53637806db 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/ShardAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/ShardAPI.java @@ -50,8 +50,6 @@ public class ShardAPI extends API { @GetMapping(value = "/shards", produces = MediaType.APPLICATION_JSON_VALUE) @ResponseBody public RestApiResponse getShards() { - - //对shards信息的统计 try { List resultShardList = new ArrayList<>(); List graphs = pdRestService.getGraphs(); diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java index 030d5de463..10c783f7db 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/StoreAPI.java @@ -81,7 +81,7 @@ public RestApiResponse getStores() { } } - // 仅支持通过该接口修改 storeState + // Only storeState can be modified through this API @PostMapping(value = "/store/{storeId}", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) @ResponseBody @@ -113,7 +113,7 @@ public String getShardGroups() { } /** - * 返回每个store上的leader + * Returns the leader on each store * * @return */ @@ -184,7 +184,7 @@ public String getStoreLog(@RequestBody TimeRangeRequest request) { @GetMapping(value = "store/{storeId}", produces = MediaType.APPLICATION_JSON_VALUE) @ResponseBody public RestApiResponse getStore(@PathVariable long storeId) { - //获取store的统计信息 + // Get the statistics of the store Metapb.Store store = null; try { store = pdRestService.getStore(storeId); @@ -238,12 +238,11 @@ public String getStoreMonitorDataText(@PathVariable long storeId) { @Data class Partition { - //分区信息 int partitionId; String graphName; String role; // shard role String workState; - long dataSize; // 占用的存储空间 + long dataSize; // The amount of storage space occupied Partition() { } @@ -262,26 +261,27 @@ class Partition { @Data class StoreStatistics { - //store的统计信息 + // store statistics long storeId; String address; String raftAddress; String version; String state; String deployPath; - String dataPath; // 数据存储路径 + String dataPath; // The path where the data is stored long startTimeStamp; - long registedTimeStamp; // 暂时取第一次心跳时间作为注册时间 - long lastHeartBeat; // 上一次心跳时间 + // For the time being, the time of the first heartbeat is taken as the registration time + long registedTimeStamp; + long lastHeartBeat; // Last heartbeat time long capacity; long available; int partitionCount; int graphSize; long keyCount; - long leaderCount; // shard role = 'Leader'的分区数量 + long leaderCount; // shard role = 'Leader' The number of partitions String serviceName; String serviceVersion; - long serviceCreatedTimeStamp; // 服务创建时间 + long serviceCreatedTimeStamp; // The time when the service was created List partitions; StoreStatistics(Metapb.Store store) { @@ -294,25 +294,26 @@ class StoreStatistics { deployPath = store.getDeployPath(); final String prefix = "file:"; if ((deployPath != null) && (deployPath.startsWith(prefix))) { - // 去掉前缀 + // Remove the prefix deployPath = deployPath.substring(prefix.length()); } if ((deployPath != null) && (deployPath.contains(".jar"))) { - // 去掉jar包之后的信息 + // Remove the information after the jar package deployPath = deployPath.substring(0, deployPath.indexOf(".jar") + 4); } dataPath = store.getDataPath(); startTimeStamp = store.getStartTimestamp(); try { serviceCreatedTimeStamp = pdRestService.getStore(store.getId()) - .getStats().getStartTime(); // 实例时间 + .getStats() + .getStartTime(); // Instance time final int base = 1000; - serviceCreatedTimeStamp *= base; // 转化为毫秒 + serviceCreatedTimeStamp *= base; // Translates to milliseconds } catch (PDException e) { e.printStackTrace(); serviceCreatedTimeStamp = store.getStartTimestamp(); } - registedTimeStamp = store.getStartTimestamp(); // 注册时间 + registedTimeStamp = store.getStartTimestamp(); // Time of registration lastHeartBeat = store.getLastHeartbeat(); capacity = store.getStats().getCapacity(); available = store.getStats().getAvailable(); @@ -320,14 +321,17 @@ class StoreStatistics { serviceName = address + "-store"; serviceVersion = store.getVersion(); List graphStatsList = store.getStats().getGraphStatsList(); - List partitionStatsList = new ArrayList<>(); // 保存分区信息 - HashSet graphNameSet = new HashSet<>(); // 用于统计图的数量 - HashSet leaderPartitionIds = new HashSet(); // 统计leader的分区数量 - // 构造分区信息(store中存储的图信息) + // Save the partition information + List partitionStatsList = new ArrayList<>(); + // The number used for the chart + HashSet graphNameSet = new HashSet<>(); + // Statistics on the number of partitions of the leader + HashSet leaderPartitionIds = new HashSet(); + // Construct partition information (graph information stored in the store) Map partition2KeyCount = new HashMap<>(); for (Metapb.GraphStats graphStats : graphStatsList) { String graphName = graphStats.getGraphName(); - // 图名只保留/g /m /s前面的部分 + // Only the part in front of /g /m /s is retained in the title final int postfixLength = 2; graphNameSet.add(graphName.substring(0, graphName.length() - postfixLength)); if ((graphStats.getGraphName() != null) && @@ -335,7 +339,7 @@ class StoreStatistics { Partition pt = new Partition(graphStats); partitionStatsList.add(pt); } - // 统计每个分区的keyCount + // Count the keyCount of each partition partition2KeyCount.put(graphStats.getPartitionId(), graphStats.getApproximateKeys()); if (graphStats.getRole() == Metapb.ShardRole.Leader) { diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/KvServiceGrpcImpl.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/KvServiceGrpcImpl.java index ffa8cdaab8..088403fb5a 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/KvServiceGrpcImpl.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/KvServiceGrpcImpl.java @@ -56,7 +56,7 @@ import lombok.extern.slf4j.Slf4j; /** - * kv 存储的核心实现类 + * The core implementation class of KV storage */ @Slf4j @GRpcService @@ -87,7 +87,7 @@ public void init() { } /** - * 普通的 put + * Ordinary put * * @param request * @param responseObserver @@ -119,7 +119,7 @@ public void put(Kv request, StreamObserver responseObserver) { } /** - * 普通的 get + * Ordinary get * * @param request * @param responseObserver @@ -151,7 +151,7 @@ public void get(K request, StreamObserver responseObserver) { } /** - * 普通的 delete + * Ordinary delete * * @param request * @param responseObserver @@ -185,7 +185,7 @@ public void delete(K request, StreamObserver responseObserver) { } /** - * 按前缀删除 + * Delete by prefix * * @param request * @param responseObserver @@ -223,7 +223,7 @@ public void deletePrefix(K request, StreamObserver responseObserver) } /** - * 按前缀查询 + * Search by prefix * * @param request * @param responseObserver @@ -253,7 +253,7 @@ public void scanPrefix(K request, StreamObserver responseObs } /** - * 获取随机非 0 字符串做 Id + * Obtain a random non-0 string as an Id * * @return */ @@ -268,7 +268,7 @@ private long getRandomLong() { } /** - * 普通的 watch + * Ordinary watch * * @param request * @param responseObserver @@ -295,7 +295,7 @@ public void watch(WatchRequest request, StreamObserver responseOb } /** - * 普通的前缀 watch + * Ordinary prefix watch * * @param request * @param responseObserver @@ -322,7 +322,7 @@ public void watchPrefix(WatchRequest request, StreamObserver resp } /** - * 上面两个方法的通用方式 + * A generic approach to the above two methods * * @param request * @param responseObserver @@ -358,7 +358,7 @@ private void clientWatch(WatchRequest request, StreamObserver res } /** - * 加锁 + * Locking * * @param request * @param responseObserver @@ -450,7 +450,7 @@ public void isLocked(LockRequest request, StreamObserver responseO } /** - * 解锁 + * Unlock * * @param request * @param responseObserver @@ -484,7 +484,7 @@ public void unlock(LockRequest request, StreamObserver responseObs } /** - * 锁续活 + * Lock renewal * * @param request * @param responseObserver @@ -520,7 +520,7 @@ public void keepAlive(LockRequest request, StreamObserver response } /** - * 带超时时间的 put + * PUT with timeout * * @param request * @param responseObserver @@ -549,7 +549,7 @@ public void putTTL(TTLRequest request, StreamObserver responseObser } /** - * 续活带有超时时间的 key + * Reactivate the key with a timeout period * * @param request * @param responseObserver diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDRestService.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDRestService.java index ed902208c8..9df8381112 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDRestService.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDRestService.java @@ -64,7 +64,7 @@ public class PDRestService implements InitializingBean { private StoreMonitorDataService storeMonitorDataService; /** - * 初始化 + * initialize * * @throws Exception */ diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDService.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDService.java index 447e599ef4..25b3f74295 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDService.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDService.java @@ -150,7 +150,7 @@ public LogService getLogService() { //} /** - * 初始化 + * initialize */ @PostConstruct public void init() throws PDException { @@ -160,7 +160,7 @@ public void init() throws PDException { RaftEngine.getInstance().addStateListener(this); RaftEngine.getInstance().addStateListener(configService); RaftEngine.getInstance().init(pdConfig.getRaft()); - //pdConfig = configService.loadConfig(); onLeaderChanged 中加载 + //pdConfig = configService.loadConfig(); onLeaderChanged storeNodeService = new StoreNodeService(pdConfig); partitionService = new PartitionService(pdConfig, storeNodeService); taskService = new TaskScheduleService(pdConfig, storeNodeService, partitionService); @@ -173,7 +173,7 @@ public void init() throws PDException { RaftEngine.getInstance().addStateListener(partitionService); pdConfig.setIdService(idService); - // 接收心跳消息 + // Receive a heartbeat message PDPulseSubject.listenPartitionHeartbeat(new PulseListener() { @Override public void onNext(PartitionHeartbeatRequest request) throws Exception { @@ -192,7 +192,7 @@ public void onCompleted() { }); /** - * 监听分区指令，并转发给 Store + // Listen for partition commands and forward them to Store */ partitionService.addInstructionListener(new PartitionInstructionListener() { private PartitionHeartbeatResponse.Builder getBuilder(Metapb.Partition partition) throws @@ -254,7 +254,7 @@ public void changePartitionKeyRange(Metapb.Partition partition, }); /** - * 监听分区状态改变消息，并转发给 Client + // Listen for partition status change messages and forward them to Client */ partitionService.addStatusListener(new PartitionStatusListener() { @Override @@ -295,7 +295,7 @@ public void onShardListOp(Metapb.ShardGroup shardGroup) { }); /** - * 监听 store 状态改变消息，并转发给 Client + // Listen for store status change messages and forward them to Client */ storeNodeService.addStatusListener(new StoreStatusListener() { @@ -343,7 +343,7 @@ public void onStoreRaftChanged(Metapb.Store store) { /** * 
-     * 注册 store，首次注册会生成新的 store_id，store_id 是 store 唯一标识
+     * Register a store, and the first registration generates a new store_id, store_id is the unique identifier of the store
      *
*/ @Override @@ -363,14 +363,13 @@ public void registerStore(Pdpb.RegisterStoreRequest request, response = Pdpb.RegisterStoreResponse.newBuilder().setHeader(newErrorHeader(e)).build(); log.error("registerStore exception: ", e); } - // 拉取所有分区信息，并返回 observer.onNext(response); observer.onCompleted(); } /** - * 根据 store_id 查找 store + * Find the store based on store_id */ @Override public void getStore(Pdpb.GetStoreRequest request, @@ -395,7 +394,7 @@ public void getStore(Pdpb.GetStoreRequest request, /** * 
-     * 修改 Store 状态等信息。
+     * Modify information such as the status of the store.
      *
*/ @Override @@ -409,10 +408,10 @@ public void setStore(Pdpb.SetStoreRequest request, try { Metapb.StoreState state = request.getStore().getState(); Long storeId = request.getStore().getId(); - // 处于 Pending 状态，才可以上线 + // In the Pending state, you can go online Metapb.Store lastStore = storeNodeService.getStore(request.getStore().getId()); if (lastStore == null) { - // storeId 不存在，抛出异常 + // storeId does not exist, an exception is thrown throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE, String.format("Store id %d does not exist!", storeId)); } @@ -437,7 +436,8 @@ public void setStore(Pdpb.SetStoreRequest request, } } logService.insertLog(LogService.NODE_CHANGE, LogService.GRPC, request.getStore()); - // 检查失败，状态改为 Pending，把错误原因返回去 + // If the check fails, the status will be changed to Pending, and the reason for the + // error will be returned if (state.equals(Metapb.StoreState.Up)) { int cores = 0; long id = request.getStore().getId(); @@ -445,7 +445,8 @@ public void setStore(Pdpb.SetStoreRequest request, int nodeCount = 0; for (Metapb.Store store : stores) { if (store.getId() == id) { - // 获取之前注册的 store 中的 cores 作为验证参数 + // Get the cores from the previously registered store as a validation + // parameter cores = store.getCores(); } if (store.getState().equals(Metapb.StoreState.Up)) { @@ -465,7 +466,8 @@ public void setStore(Pdpb.SetStoreRequest request, } } Metapb.Store store = request.getStore(); - // 下线之前先判断一下，活跃机器数是否大于最小阈值 + // Before going offline, check whether the number of active machines is greater than + // the minimum threshold if (state.equals(Metapb.StoreState.Tombstone)) { List activeStores = storeNodeService.getActiveStores(); if (lastStore.getState() == Metapb.StoreState.Up @@ -479,7 +481,7 @@ public void setStore(Pdpb.SetStoreRequest request, "check activeStores or online shardsList size"); } if (lastStore.getState() == Metapb.StoreState.Exiting) { - // 如果已经是下线中的状态，则不作进一步处理 + // If it is already in the offline state, no further processing will be made throw new PDException(Pdpb.ErrorType.Store_Tombstone_Doing_VALUE, "Downline is in progress, do not resubmit"); } @@ -488,15 +490,16 @@ public void setStore(Pdpb.SetStoreRequest request, if (resultMap.get("current_store_is_online") != null && (boolean) resultMap.get("current_store_is_online")) { log.info("updateStore removeActiveStores store {}", store.getId()); - // 将在线的 store 的状态设置为下线中，等待副本迁移 + // Set the status of the online store to Offline and wait for the replica + // to be migrated store = Metapb.Store.newBuilder(lastStore) .setState(Metapb.StoreState.Exiting).build(); - // 进行分区迁移操作 + // Perform partition migration operations taskService.movePartitions((Map>) resultMap.get( "movedPartitions")); } else { - // store 已经离线的，不做副本迁移 - // 将状态改为 Tombstone + // If the store is offline, the replica is not migrated + // Change the status to Tombstone } } else { throw new PDException(Pdpb.ErrorType.UPDATE_STORE_STATE_ERROR_VALUE, @@ -505,7 +508,6 @@ public void setStore(Pdpb.SetStoreRequest request, "the partitions of current store!"); } } - // 替换 license 都走 grpc store = storeNodeService.updateStore(store); response = Pdpb.SetStoreResponse.newBuilder().setHeader(okHeader).setStore(store).build(); @@ -518,9 +520,6 @@ public void setStore(Pdpb.SetStoreRequest request, observer.onCompleted(); } - /** - * 返回所有的 store,exclude_offline_stores=true，返回活跃的 stores - */ @Override public void getAllStores(Pdpb.GetAllStoresRequest request, io.grpc.stub.StreamObserver observer) { @@ -548,7 +547,7 @@ public void getAllStores(Pdpb.GetAllStoresRequest request, } /** - * 处理 store 心跳 + * Handle store heartbeats */ @Override public void storeHeartbeat(Pdpb.StoreHeartbeatRequest request, @@ -595,7 +594,7 @@ public void storeHeartbeat(Pdpb.StoreHeartbeatRequest request, /** * 
-     * 查找 key 所属的分区
+     * Find the partition to which the key belongs
      *
*/ @Override @@ -624,7 +623,7 @@ public void getPartition(Pdpb.GetPartitionRequest request, /** * 
-     * 查找 HashCode 所属的分区
+     * Find the partition to which the HashCode belongs
      *
*/ @Override @@ -651,7 +650,7 @@ public void getPartitionByCode(Pdpb.GetPartitionByCodeRequest request, } /** - * 根据 partition_id 查找 partition + * Find partition based on partition_id */ @Override public void getPartitionByID(Pdpb.GetPartitionByIDRequest request, @@ -684,7 +683,7 @@ public void getPartitionByID(Pdpb.GetPartitionByIDRequest request, /** * 
-     * 更新分区信息，主要用来更新分区 key 范围，调用此接口需谨慎，否则会造成数据丢失。
+     * Update partition information, mainly used to update the partition key range, call this API with caution, otherwise it will cause data loss.
      *
*/ @Override @@ -709,7 +708,7 @@ public void updatePartition(Pdpb.UpdatePartitionRequest request, } /** - * 根据 partition_id 查找 partition + * Find partition based on partition_id */ @Override public void delPartition(Pdpb.DelPartitionRequest request, @@ -740,7 +739,7 @@ public void delPartition(Pdpb.DelPartitionRequest request, } /** - * 给定 key 范围查找所属的 partition 集合 + * The set of partitions to which a given key range looks */ @Override public void scanPartitions(Pdpb.ScanPartitionsRequest request, @@ -769,7 +768,7 @@ public void scanPartitions(Pdpb.ScanPartitionsRequest request, } /** - * 获得图信息 + * Get graph information */ @Override public void getGraph(GetGraphRequest request, @@ -800,7 +799,7 @@ public void getGraph(GetGraphRequest request, } /** - * 修改图信息 + * Modify the diagram information */ @Override public void setGraph(Pdpb.SetGraphRequest request, @@ -824,7 +823,7 @@ public void setGraph(Pdpb.SetGraphRequest request, } /** - * 获得图信息 + * Get graph information */ @Override public void delGraph(Pdpb.DelGraphRequest request, @@ -852,7 +851,7 @@ public void delGraph(Pdpb.DelGraphRequest request, /** * 
-     * 根据条件查询分区信息，包括 Store、Graph 等条件
+     * Query partition information based on conditions, such as Store and Graph
      *
*/ @Override @@ -862,7 +861,8 @@ public void queryPartitions(Pdpb.QueryPartitionsRequest request, redirectToLeader(PDGrpc.getQueryPartitionsMethod(), request, observer); return; } - //TODO 临时采用遍历方案，后续使用 rocksdb 存储时，通过 kv 索引实现 + // The traversal scheme is used temporarily, and when the rocksdb storage is used in + // the future, it is implemented through KV indexes Metapb.PartitionQuery query = request.getQuery(); List partitions = partitionService.getPartitions(query.getGraphName()); List result = new ArrayList<>(); @@ -943,7 +943,7 @@ public void resetId(Pdpb.ResetIdRequest request, } /** - * 获取集群成员信息 + * Obtain cluster member information */ @Override public void getMembers(Pdpb.GetMembersRequest request, @@ -992,7 +992,7 @@ public void getStoreStatus(Pdpb.GetAllStoresRequest request, } /** - * 读取 PD 配置 + * Read the PD configuration */ @Override public void getPDConfig(Pdpb.GetPDConfigRequest request, @@ -1016,7 +1016,7 @@ public void getPDConfig(Pdpb.GetPDConfigRequest request, } /** - * 修改 PD 配置 + * Modify the PD configuration */ @Override public void setPDConfig(Pdpb.SetPDConfigRequest request, @@ -1028,13 +1028,13 @@ public void setPDConfig(Pdpb.SetPDConfigRequest request, Pdpb.SetPDConfigResponse response = null; try { if (request.getPdConfig().getShardCount() % 2 != 1) { - // 副本数奇偶校验 + // Parity of the number of replicas throw new PDException(Pdpb.ErrorType.SET_CONFIG_SHARD_COUNT_ERROR_VALUE, "shard count must be an odd number!"); } if (request.getPdConfig().getShardCount() > storeNodeService.getActiveStores().size()) { - // 不能大于活跃的 store 数量 + // It can't be greater than the number of active stores throw new PDException(Pdpb.ErrorType.SET_CONFIG_SHARD_COUNT_ERROR_VALUE, "shard count can't be greater than the number of active " + "stores!"); @@ -1042,7 +1042,8 @@ public void setPDConfig(Pdpb.SetPDConfigRequest request, int oldShardCount = configService.getPDConfig().getShardCount(); int newShardCount = request.getPdConfig().getShardCount(); if (newShardCount > oldShardCount) { - // 如果副本数增大，则检查 store 内部的资源是否够用 + // If the number of replicas increases, check whether the resources inside the + // store are sufficient if (!isResourceEnough(oldShardCount, newShardCount)) { throw new PDException(Pdpb.ErrorType.SET_CONFIG_SHARD_COUNT_ERROR_VALUE, "There is not enough disk space left!"); @@ -1063,7 +1064,7 @@ public void setPDConfig(Pdpb.SetPDConfigRequest request, } /** - * 读取图空间配置 + * Read the graph space configuration */ @Override public void getGraphSpace(Pdpb.GetGraphSpaceRequest request, @@ -1086,7 +1087,7 @@ public void getGraphSpace(Pdpb.GetGraphSpaceRequest request, } /** - * 修改图空间配置 + * Modify the graph space configuration */ @Override public void setGraphSpace(Pdpb.SetGraphSpaceRequest request, @@ -1108,7 +1109,7 @@ public void setGraphSpace(Pdpb.SetGraphSpaceRequest request, /** * 
-     * 数据分裂
+     * Data fragmentation
      *
*/ @Override @@ -1154,7 +1155,7 @@ public void splitGraphData(Pdpb.SplitGraphDataRequest request, } /** - * 在 store 之间平衡数据 + * Balance data between stores */ @Override public void movePartition(Pdpb.MovePartitionRequest request, @@ -1179,7 +1180,7 @@ public void movePartition(Pdpb.MovePartitionRequest request, /** * 
-     * 获取集群健康状态
+     * Obtain the cluster health status
      *
*/ @Override @@ -1199,7 +1200,7 @@ public void getClusterStats(Pdpb.GetClusterStatsRequest request, /** * 
-     * 汇报分区分裂等任务执行结果
+     * Report the results of tasks such as partition splitting
      *
*/ @Override @@ -1280,7 +1281,7 @@ public boolean isLeader() { //} /** - * 更新 peerList + * Renewal peerList */ @Override public void changePeerList(Pdpb.ChangePeerListRequest request, @@ -1395,7 +1396,6 @@ public void delStore(Pdpb.DetStoreRequest request, try { Metapb.Store store = storeNodeService.getStore(storeId); if (Metapb.StoreState.Tombstone == store.getState()) { - // 只有已经被下线 (Tombstone) 的 store 可以被删除 storeNodeService.removeStore(storeId); response = Pdpb.DetStoreResponse.newBuilder() .setHeader(okHeader) @@ -1438,35 +1438,35 @@ private boolean checkShardCount(int newShardCount) { } /** - * 检查 store 资源是否够用 + * Check that the store resources are sufficient */ public boolean isResourceEnough(int oldShardCount, int newShardCount) { - // 活跃的 store 的资源是否够用 + // Whether the resources of the active store are sufficient try { - - float expansionRatio = newShardCount / oldShardCount; // 占用的存储空间膨胀的倍数 - // 当前占用的空间 + // The multiple of the storage space occupied + float expansionRatio = newShardCount / oldShardCount; + // The space currently occupied long currentDataSize = 0L; - // 数据膨胀后占用的空间 + // The space occupied after data bloat long newDataSize = 0L; - // 总的可用空间 + // Total free space long totalAvaible = 0L; - // 统计当前占用的存储空间 + // Statistics on the current storage space for (Metapb.Store store : storeNodeService.getStores()) { List graphStatsList = store.getStats().getGraphStatsList(); for (Metapb.GraphStats graphStats : graphStatsList) { currentDataSize += graphStats.getApproximateSize(); } } - // 估计数据膨胀后占用的存储空间 + // Estimate the storage space consumed after data bloat newDataSize = (long) Math.ceil(currentDataSize * expansionRatio); - // 统计所有活跃的 store 里面可用的空间 + // Count the available space in all active stores List activeStores = storeNodeService.getActiveStores(); for (Metapb.Store store : activeStores) { Metapb.StoreStats storeStats = store.getStats(); totalAvaible += storeStats.getAvailable(); } - // 考虑当分区均匀分配的情况下，资源是否可用 + // Consider whether resources are available when partitions are evenly distributed return totalAvaible > newDataSize - currentDataSize; } catch (PDException e) { e.printStackTrace(); @@ -1476,7 +1476,7 @@ public boolean isResourceEnough(int oldShardCount, int newShardCount) { /** * 
-     * 对 rocksdb 进行 compaction
+     * Compaction on rocksdb
      *
*/ @Override diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/UpgradeService.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/UpgradeService.java index f99efe5e01..40f3d2ef89 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/UpgradeService.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/UpgradeService.java @@ -51,13 +51,13 @@ public void upgrade() throws PDException { var dataVersion = getDataVersion(); log.info("now db data version : {}", dataVersion); for (VersionUpgradeScript script : factory.getScripts()) { - // 执行过，run once的跳过 + // Executed, run once skipped if (isExecuted(script.getClass().getName()) && script.isRunOnce()) { log.info("Script {} is Executed and is run once", script.getClass().getName()); continue; } - // 判断跳过的条件 + // Determine the conditions for skipping if (dataVersion == null && !script.isRunWithoutDataVersion() || dataVersion != null && !versionCompare( dataVersion, diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionUpgradeScript.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionUpgradeScript.java index d90079043b..5cc913fd1a 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionUpgradeScript.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionUpgradeScript.java @@ -40,7 +40,8 @@ public interface VersionUpgradeScript { String getLowVersion(); /** - * pd中没有data version的时候，是否执行. 一般是对应3。6。2之前的版本 + * If there is no data version in the PD, whether to execute the . Generally, it corresponds + * to 3.6.2 previous versions * * @return run when pd has no data version */ diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/watch/KvWatchSubject.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/watch/KvWatchSubject.java index f0109a6235..a1a297014b 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/watch/KvWatchSubject.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/watch/KvWatchSubject.java @@ -41,7 +41,7 @@ import lombok.extern.slf4j.Slf4j; /** - * watch订阅、响应处理类 + * Watch subscription and response processing classes **/ @Slf4j public class KvWatchSubject { @@ -57,7 +57,7 @@ public class KvWatchSubject { BiPredicate startWith = String::startsWith; /** - * 会使用以下三组key: + * The following three sets of keys will be used: * clients -> W@KW@key@clientId * rocksdb key1 ->W@KW@key@clientId * rocksdb key2 ->W@clientId@KW@key@clientId @@ -86,12 +86,13 @@ private void removeWatchKey(String key, String delimiter, long clientId) throws } /** - * 增加观察者 + * Increase observers * - * @param key 观察的key - * @param clientId 客户端标识 + * @param key The key of the observation + * @param clientId Client identity * @param observer - * @param delimiter 观察类型标识符，对前缀监听或者对key的监听可以通过此参数区分 + * @param delimiter Observe the type identifier, listen to the prefix or listen to the key + * can be distinguished by this parameter * @throws PDException */ public void addObserver(String key, long clientId, StreamObserver observer, @@ -111,11 +112,13 @@ public void removeObserver(String key, long clientId, String delimiter) throws P } /** - * 通知观察者方法，key和prefix都使用此方法，predicate不同 + * The notification observer method, which is used by both key and prefix, is different from + * predicate * * @param key - * @param watchType 观察类型，一般是增加和删除 - * @param predicate 判断等于或者是前匹配，用来适配key或prefix观察 + * @param watchType Observation types, generally additions and deletions + * @param predicate Determine whether it is equal or pre-matched, and use it to adapt to the + * key or prefix observation * @param kvs * @throws PDException */ @@ -177,10 +180,10 @@ public void notifyAllObserver(String key, WatchType watchType, WatchKv[] kvs) th } /** - * 续活客户端 - * 1.往客户端发一个alive的消息，带重试哈 - * 2.如果有响应，则续活之前保存的那两组key - * 3.如果多次都失败，则删除内存和rocksdb的数据 + * Renew the client + * 1. Send an alive message to the client with a retry + * 2. If there is a response, the two sets of keys saved before will be reactivated + * 3. If it fails multiple times, delete the data of memory and rocksdb */ public void keepClientAlive() { WatchResponse testAlive = WatchResponse.newBuilder().setState(WatchState.Alive).build(); @@ -256,7 +259,7 @@ private void removeClient(StreamObserver value, String key, Strin } /** - * 通知客户端leader切换了，重连 + * Notify the client that the leader has switched and reconnect */ public void notifyClientChangeLeader() { WatchResponse response = diff --git a/hugegraph-pd/hg-pd-service/src/main/resources/application.yml b/hugegraph-pd/hg-pd-service/src/main/resources/application.yml index 25471b6cce..87a33273b7 100644 --- a/hugegraph-pd/hg-pd-service/src/main/resources/application.yml +++ b/hugegraph-pd/hg-pd-service/src/main/resources/application.yml @@ -31,7 +31,7 @@ management: grpc: port: 8686 - # grpc的服务地址 + # GRPC's service address host: 127.0.0.1 netty-server: max-inbound-message-size: 100MB @@ -44,37 +44,38 @@ server: port: 8620 pd: - # 定期检查集群是否健康的时间间隔，单位秒 + # Periodically check whether the cluster is healthy at intervals, in seconds patrol-interval: 300 - # 存储路径 + # Storage path data-path: tmp/pd/8610 - # 最少节点数，少于该数字，集群停止入库 + # If the minimum number of nodes is less than this number, the cluster will stop being stored initial-store-count: 1 - # 初始store列表，在列表内的store自动激活 + # The initial store list is automatically activated in the store initial-store-list: 127.0.0.1:8502 raft: - # 本机raft服务地址 + # The address of the local raft service address: 127.0.0.1:8610 - # PD集群服务地址 + # The service address of the PD cluster peers-list: 127.0.0.1:8610,127.0.0.1:8611,127.0.0.1:8612 - # raft rpc读写超时时间,单位毫秒 + # The read and write timeout period of the raft rpc, in milliseconds rpc-timeout: 10000 - # 快照生成时间间隔，单位秒 + # The interval between snapshot generation, in seconds snapshotInterval: 300 metrics: true store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 + # If the store heartbeat timeout period exceeds this time, the store is temporarily unavailable and the leader is transferred to another replica in seconds keepAlive-timeout: 300 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 + # The time when the store went offline. After that time, the store is considered permanently unavailable, and the replica is allocated to another machine, in seconds max-down-time: 86400 partition: - # 默认每个分区副本数 + # Default number of replicas per partition default-shard-count: 3 - # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count + # The default maximum number of replicas per machine, the initial number of partitions= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 12 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 + # After the client registers, the maximum number of heartbeats is not reached, the previous + # registration information will be deleted heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-service/src/test/resources/application-server0.yml b/hugegraph-pd/hg-pd-service/src/test/resources/application-server0.yml index 5e1d63e943..5ee5108cf6 100644 --- a/hugegraph-pd/hg-pd-service/src/test/resources/application-server0.yml +++ b/hugegraph-pd/hg-pd-service/src/test/resources/application-server0.yml @@ -41,31 +41,21 @@ pd: patrol-interval: 3000000 data-path: tmp/8686 - # 最少节点数，少于该数字，集群停止入库 initial-store-count: 1 - # 初始store列表，在列表内的store自动激活 initial-store-list: 127.0.0.1:8500 raft: address: 127.0.0.1:8610 - # raft集群 peers-list: 127.0.0.1:8610 - # raft rpc读写超时时间,单位毫秒 rpc-timeout: 10000 - # 快照生成时间间隔，单位秒 snapshotInterval: 30000 metrics: true store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 keepAlive-timeout: 300 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 max-down-time: 180000 partition: - # 默认每个分区副本数 default-shard-count: 3 - # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 12 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-service/src/test/resources/application-server1.yml b/hugegraph-pd/hg-pd-service/src/test/resources/application-server1.yml index 7cb53fe1c8..11b6a722c4 100644 --- a/hugegraph-pd/hg-pd-service/src/test/resources/application-server1.yml +++ b/hugegraph-pd/hg-pd-service/src/test/resources/application-server1.yml @@ -41,31 +41,20 @@ pd: patrol-interval: 3000000 data-path: tmp/8686 - # 最少节点数，少于该数字，集群停止入库 initial-store-count: 1 - # 初始store列表，在列表内的store自动激活 initial-store-list: 127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503 - #initial-store-list: 127.0.0.1:8501 raft: address: 127.0.0.1:8610 - # raft集群 peers-list: 127.0.0.1:8610 - # raft rpc读写超时时间,单位毫秒 rpc-timeout: 10000 - # 快照生成时间间隔，单位秒 snapshotInterval: 30000 metrics: true store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 keepAlive-timeout: 300 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 max-down-time: 180000 partition: - # 默认每个分区副本数 default-shard-count: 3 - # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 6 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-service/src/test/resources/application-server2.yml b/hugegraph-pd/hg-pd-service/src/test/resources/application-server2.yml index 5e1dd50a98..7fb2e362b0 100644 --- a/hugegraph-pd/hg-pd-service/src/test/resources/application-server2.yml +++ b/hugegraph-pd/hg-pd-service/src/test/resources/application-server2.yml @@ -39,7 +39,6 @@ server: port: 8621 pd: - # 集群ID，区分不同的PD集群 cluster_id: 1 patrol-interval: 300000 data-path: tmp/8687 @@ -48,26 +47,17 @@ pd: raft: enable: true address: 127.0.0.1:8611 - # raft集群 peers-list: 127.0.0.1:8610,127.0.0.1:8611,127.0.0.1:8612 - # raft rpc读写超时时间,单位毫秒 rpc-timeout: 10000 - # 快照生成时间间隔，单位秒 snapshotInterval: 300 metrics: true - # 初始store列表，在列表内的store自动激活 initial-store-list: 127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503 store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 keepAlive-timeout: 300 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 max-down-time: 1800 partition: - # 默认每个分区副本数 default-shard-count: 3 - # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 3 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-service/src/test/resources/application-server3.yml b/hugegraph-pd/hg-pd-service/src/test/resources/application-server3.yml index d2b88950ab..b2470a315a 100644 --- a/hugegraph-pd/hg-pd-service/src/test/resources/application-server3.yml +++ b/hugegraph-pd/hg-pd-service/src/test/resources/application-server3.yml @@ -39,7 +39,6 @@ server: port: 8622 pd: - # 集群ID，区分不同的PD集群 cluster_id: 1 patrol-interval: 300000 data-path: tmp/8688 @@ -48,26 +47,17 @@ pd: raft: enable: true address: 127.0.0.1:8612 - # raft集群 peers-list: 127.0.0.1:8610,127.0.0.1:8611,127.0.0.1:8612 - # raft rpc读写超时时间,单位毫秒 rpc-timeout: 10000 - # 快照生成时间间隔，单位秒 snapshotInterval: 300 metrics: true - # 初始store列表，在列表内的store自动激活 initial-store-list: 127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503 store: - # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 keepAlive-timeout: 300 - # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 max-down-time: 1800 partition: - # 默认每个分区副本数 default-shard-count: 3 - # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count store-max-shard-count: 3 discovery: - #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java index c745f42355..3aedfb117a 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java @@ -318,8 +318,6 @@ public void testSplitData() { Metapb.PDConfig config = pdClient.getPDConfig(); pdClient.setPDConfig(config.toBuilder().setMaxShardsPerStore(12).build()); System.out.println(pdClient.getPDConfig()); - - // 开始分区分裂 pdClient.splitData(); } catch (Exception e) { e.printStackTrace(); diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java index ebd7d89fc2..d9dd0d40eb 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java @@ -244,7 +244,6 @@ public void testRange2() { this.cache.updatePartition(partition2); System.out.println(this.cache.debugCacheByGraphName("graph0")); - // 中间有缺失 var partition3 = createPartition(1, "graph0", 2, 3); this.cache.updatePartition(partition3); System.out.println(this.cache.debugCacheByGraphName("graph0"));