Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: improve rocksdb code #2002

Merged
merged 2 commits into from
Nov 3, 2022
Merged

refactor: improve rocksdb code #2002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
183b940
refactor: improve rocksdb code
Nov 2, 2022
40a30f1
improve
Nov 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -924,26 +924,26 @@ protected static boolean indexFieldValuesUnmatched(byte[] value,
return false;
}

public static final byte[] increaseOne(byte[] bytes) {
public static void increaseOne(byte[] bytes) {
final byte BYTE_MAX_VALUE = (byte) 0xff;
final byte INCREASE_STEP = 0x01;
assert bytes.length > 0;
byte last = bytes[bytes.length - 1];
if (last != BYTE_MAX_VALUE) {
bytes[bytes.length - 1] += 0x01;
bytes[bytes.length - 1] += INCREASE_STEP;
} else {
// Process overflow (like [1, 255] => [2, 0])
int i = bytes.length - 1;
for (; i > 0 && bytes[i] == BYTE_MAX_VALUE; --i) {
bytes[i] += 0x01;
bytes[i] += INCREASE_STEP;
}
if (bytes[i] == BYTE_MAX_VALUE) {
assert i == 0;
throw new BackendException("Unable to increase bytes: %s",
Bytes.toHex(bytes));
}
bytes[i] += 0x01;
bytes[i] += INCREASE_STEP;
}
return bytes;
}

@Override
Expand Down
10 changes: 5 additions & 5 deletions ...h-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,11 +744,11 @@ public static TableFormatConfig initTableConfig(HugeConfig conf) {
return tableConfig;
}

public static final byte[] encode(String string) {
public static byte[] encode(String string) {
return StringEncoding.encode(string);
}

public static final String decode(byte[] bytes) {
public static String decode(byte[] bytes) {
return StringEncoding.decode(bytes);
}

Expand All @@ -758,7 +758,7 @@ public static final String decode(byte[] bytes) {
private final class StdSession extends RocksDBSessions.Session {

private WriteBatch batch;
private WriteOptions writeOptions;
private final WriteOptions writeOptions;

public StdSession(HugeConfig conf) {
this.batch = new WriteBatch();
Expand Down Expand Up @@ -955,8 +955,8 @@ public void deleteSingle(String table, byte[] key) {
@Override
public void deletePrefix(String table, byte[] key) {
byte[] keyFrom = key;
byte[] keyTo = Arrays.copyOf(key, key.length);
keyTo = BinarySerializer.increaseOne(keyTo);
byte[] keyTo = Arrays.copyOf(keyFrom, keyFrom.length);
BinarySerializer.increaseOne(keyTo);
try (CFHandle cf = cf(table)) {
this.batch.deleteRange(cf.get(), keyFrom, keyTo);
} catch (RocksDBException e) {
Expand Down
37 changes: 19 additions & 18 deletions hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand Down Expand Up @@ -182,7 +183,7 @@ protected List<String> olapTables() {
}

protected List<String> tableNames(HugeType type) {
return type != HugeType.OLAP ? Arrays.asList(this.table(type).table()) :
return type != HugeType.OLAP ? Collections.singletonList(this.table(type).table()) :
this.olapTables();
}

Expand Down Expand Up @@ -287,7 +288,7 @@ private void shutdownOpenPool(ExecutorService openPool) {
"open-pool");
}

boolean terminated = false;
boolean terminated;
openPool.shutdown();
try {
terminated = openPool.awaitTermination(OPEN_TIMEOUT,
Expand Down Expand Up @@ -803,24 +804,11 @@ private void useSessions() {
}
}

private void closeSessions() {
Iterator<Map.Entry<String, RocksDBSessions>> iter = this.dbs.entrySet()
.iterator();
while (iter.hasNext()) {
Map.Entry<String, RocksDBSessions> entry = iter.next();
RocksDBSessions sessions = entry.getValue();
boolean closed = sessions.close();
if (closed) {
iter.remove();
}
}
}

private List<Session> session() {
this.checkOpened();

if (this.tableDiskMapping.isEmpty()) {
return Arrays.asList(this.sessions.session());
return Collections.singletonList(this.sessions.session());
}

// Collect session of each table with optimized disk
Expand All @@ -832,6 +820,19 @@ private List<Session> session() {
return list;
}

private void closeSessions() {
Iterator<Map.Entry<String, RocksDBSessions>> iter = this.dbs.entrySet()
.iterator();
while (iter.hasNext()) {
Map.Entry<String, RocksDBSessions> entry = iter.next();
RocksDBSessions sessions = entry.getValue();
boolean closed = sessions.close();
if (closed) {
iter.remove();
}
}
}

private Collection<RocksDBSessions> sessions() {
return this.dbs.values();
}
Expand Down Expand Up @@ -1094,7 +1095,7 @@ public void clearOlapTable(Id id) {
String name = this.olapTableName(id);
RocksDBTable table = this.table(name);
RocksDBSessions db = this.db(HugeType.OLAP);
if (table == null || !db.existsTable(table.table())) {
if (!db.existsTable(table.table())) {
throw new HugeException("Not exist table '%s''", name);
}
this.dropTable(db, table.table());
Expand All @@ -1106,7 +1107,7 @@ public void removeOlapTable(Id id) {
String name = this.olapTableName(id);
RocksDBTable table = this.table(name);
RocksDBSessions db = this.db(HugeType.OLAP);
if (table == null || !db.existsTable(table.table())) {
if (!db.existsTable(table.table())) {
throw new HugeException("Not exist table '%s''", name);
}
this.dropTable(db, table.table());
Expand Down
7 changes: 3 additions & 4 deletions hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,9 +291,8 @@ public boolean isOlap() {
return false;
}

protected static final BackendEntryIterator newEntryIterator(
BackendColumnIterator cols,
Query query) {
protected static BackendEntryIterator newEntryIterator(BackendColumnIterator cols,
Query query) {
return new BinaryEntryIterator<>(cols, query, (entry, col) -> {
if (entry == null || !entry.belongToMe(col)) {
HugeType type = query.resultType();
Expand All @@ -307,7 +306,7 @@ protected static final BackendEntryIterator newEntryIterator(
});
}

protected static final long sizeOfBackendEntry(BackendEntry entry) {
protected static long sizeOfBackendEntry(BackendEntry entry) {
return BinaryEntryIterator.sizeOfEntry(entry);
}

Expand Down
2 changes: 1 addition & 1 deletion hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ protected BackendColumnIterator queryByCond(Session session,
E.checkArgumentNotNull(min, "Range index begin key is missing");
byte[] begin = min.asBytes();
if (!minEq) {
begin = BinarySerializer.increaseOne(begin);
BinarySerializer.increaseOne(begin);
}

if (max == null) {
Expand Down
4 changes: 2 additions & 2 deletions ...ocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ public String hardLinkSnapshot(String snapshotPath)
}

@Override
public void reloadRocksDB() throws RocksDBException {
public void reloadRocksDB() {
throw new UnsupportedOperationException("reloadRocksDB");
}

Expand Down Expand Up @@ -235,7 +235,7 @@ protected synchronized void doClose() {
*/
private final class SstSession extends Session {

private Map<String, Changes> batch;
private final Map<String, Changes> batch;

public SstSession() {
this.batch = new HashMap<>();
Expand Down