Skip to content

Commit

Permalink
Added more java docs and refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@ArafatKhan2198
ArafatKhan2198 committed Mar 1, 2024
1 parent 0e1fe0d commit c86182e
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 146 deletions.
2 changes: 1 addition & 1 deletion ...ration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public void testNamespaceSummaryAPI() throws Exception {
OzoneStorageContainerManager reconSCM =
cluster.getReconServer().getReconStorageContainerManager();
NSSummaryEndpoint endpoint = new NSSummaryEndpoint(namespaceSummaryManager,
omMetadataManagerInstance, reconSCM,new OzoneConfiguration());
omMetadataManagerInstance, reconSCM);
Response basicInfo = endpoint.getBasicInfo("/vol1/bucket1/dir1");
NamespaceSummaryResponse entity =
(NamespaceSummaryResponse) basicInfo.getEntity();
Expand Down
2 changes: 0 additions & 2 deletions ...p-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@
*/
package org.apache.hadoop.ozone.recon.api.handlers;

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
Expand Down
202 changes: 59 additions & 143 deletions ...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,153 +221,29 @@ public long handleDirectKeys(long parentId, boolean withReplica,
}

if (enableFileSystemPaths) {
keyDataSizeWithReplica += handleDirectKeysForFSOLayout(
parentId, withReplica, listFile, normalizedPath, duData, keyTable,
seekPrefix, nsSummary);
keyDataSizeWithReplica +=
handleDirectKeysForFSOLayout(parentId, withReplica, listFile, duData,
keyTable, seekPrefix, nsSummary);
} else {
keyDataSizeWithReplica += handleDirectKeysForOBSLayout(
parentId, withReplica, listFile, duData, keyTable, seekPrefix);
keyDataSizeWithReplica +=
handleDirectKeysForOBSLayout(withReplica, listFile, duData, keyTable,
seekPrefix);
}

return keyDataSizeWithReplica;
}


// public long handleDirectKeysForOBSLayout(long parentId, boolean withReplica,
// boolean listFile,
// List<DUResponse.DiskUsage> duData,
// Table<String, OmKeyInfo> keyTable,
// String seekPrefix)
// throws IOException {
//
// long keyDataSizeWithReplica = 0L;
//
// try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
// iterator = keyTable.iterator()) {
// iterator.seek(seekPrefix);
//
// while (iterator.hasNext()) {
// // KeyName : OmKeyInfo-Object
// Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
// String dbKey = kv.getKey();
//
// // Exit loop if the key doesn't match the seekPrefix.
// if (!dbKey.startsWith(seekPrefix)) {
// break;
// }
//
// OmKeyInfo keyInfo = kv.getValue();
// if (keyInfo != null) {
// DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
// String objectName = keyInfo.getKeyName();
// diskUsage.setSubpath(objectName);
// diskUsage.setKey(true);
// diskUsage.setSize(keyInfo.getDataSize());
//
// if (withReplica) {
// long keyDU = keyInfo.getReplicatedSize();
// keyDataSizeWithReplica += keyDU;
// diskUsage.setSizeWithReplica(keyDU);
// }
// // List all the keys for the OBS bucket if requested.
// if (listFile) {
// duData.add(diskUsage);
// }
// }
// }
// }
//
// return keyDataSizeWithReplica;
// }

// public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
// boolean listFile,
// String normalizedPath,
// List<DUResponse.DiskUsage> duData,
// Table<String, OmKeyInfo> keyTable,
// String seekPrefix,
// NSSummary nsSummary)
// throws IOException {
//
// long keyDataSizeWithReplica = 0L;
//
// if (omBucketInfo.getObjectID() != parentId) {
// String dirName = nsSummary.getDirName();
// seekPrefix += dirName;
// }
// String[] seekKeys = seekPrefix.split(OM_KEY_PREFIX);
// try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
// iterator = keyTable.iterator()) {
//
// iterator.seek(seekPrefix);
//
// while (iterator.hasNext()) {
// Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
// String dbKey = kv.getKey();
//
// if (!dbKey.startsWith(seekPrefix)) {
// break;
// }
//
// String[] keys = dbKey.split(OM_KEY_PREFIX);
//
// // iteration moved to the next level
// // and not handling direct keys
// if (keys.length - seekKeys.length > 1) {
// continue;
// }
//
// OmKeyInfo keyInfo = kv.getValue();
// if (keyInfo != null) {
// // skip directory markers, just include directKeys
// if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
// continue;
// }
// DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
// String subpath = buildSubpath(normalizedPath,
// keyInfo.getFileName());
// diskUsage.setSubpath(subpath);
// diskUsage.setKey(true);
// diskUsage.setSize(keyInfo.getDataSize());
//
// if (withReplica) {
// long keyDU = keyInfo.getReplicatedSize();
// keyDataSizeWithReplica += keyDU;
// diskUsage.setSizeWithReplica(keyDU);
// }
// // list the key as a subpath
// if (listFile) {
// duData.add(diskUsage);
// }
// }
// }
// }
//
// return keyDataSizeWithReplica;
// }

// Create a method to generate DUResponse.DiskUsage objects
private DUResponse.DiskUsage createDiskUsage(OmKeyInfo keyInfo, boolean withReplica, boolean listFile, List<DUResponse.DiskUsage> duData) {
DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
String objectName = keyInfo.getKeyName();
diskUsage.setSubpath(objectName);
diskUsage.setKey(true);
diskUsage.setSize(keyInfo.getDataSize());

if (withReplica) {
long keyDU = keyInfo.getReplicatedSize();
diskUsage.setSizeWithReplica(keyDU);
}

if (listFile) {
duData.add(diskUsage);
}

return diskUsage;
}

// Update your existing methods to use the new createDiskUsage method
public long handleDirectKeysForOBSLayout(long parentId, boolean withReplica,
/**
* This method handles disk usage of direct keys for OBS layout.
* @param withReplica if withReplica is enabled, set sizeWithReplica
* @param listFile if listFile is enabled, append key DU as a subpath
* @param duData the current DU data
* @param keyTable the key table
* @param seekPrefix the seek prefix used to position the iterator
* @return the total DU of all direct keys
* @throws IOException
*/
public long handleDirectKeysForOBSLayout(boolean withReplica,
boolean listFile,
List<DUResponse.DiskUsage> duData,
Table<String, OmKeyInfo> keyTable,
Expand Down Expand Up @@ -402,9 +278,20 @@ public long handleDirectKeysForOBSLayout(long parentId, boolean withReplica,
return keyDataSizeWithReplica;
}

/**
* This method handles disk usage of direct keys for OBS layout.
* @param parentId parent directory/bucket
* @param withReplica if withReplica is enabled, set sizeWithReplica
* @param listFile if listFile is enabled, append key DU as a subpath
* @param duData the current DU data
* @param keyTable the key table
* @param seekPrefix the seek prefix used to position the iterator
* @param nsSummary of the parent directory/bucket
* @return the total DU of all direct keys
* @throws IOException
*/
public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
boolean listFile,
String normalizedPath,
List<DUResponse.DiskUsage> duData,
Table<String, OmKeyInfo> keyTable,
String seekPrefix,
Expand All @@ -418,7 +305,8 @@ public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
}
String[] seekKeys = seekPrefix.split(OM_KEY_PREFIX);
try (
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator = keyTable.iterator()) {
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
iterator = keyTable.iterator()) {

iterator.seek(seekPrefix);

Expand Down Expand Up @@ -456,6 +344,34 @@ public long handleDirectKeysForFSOLayout(long parentId, boolean withReplica,
return keyDataSizeWithReplica;
}

/**
* This method handles disk usage calculation for legacy buckets.
* @param keyInfo the key info
* @param withReplica if withReplica is enabled, set sizeWithReplica
* @param listFile if listFile is enabled, append key DU as a subpath
*/
private DUResponse.DiskUsage createDiskUsage(OmKeyInfo keyInfo,
boolean withReplica,
boolean listFile,
List<DUResponse.DiskUsage> duData) {
DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
String objectName = keyInfo.getKeyName();
diskUsage.setSubpath(objectName);
diskUsage.setKey(true);
diskUsage.setSize(keyInfo.getDataSize());

if (withReplica) {
long keyDU = keyInfo.getReplicatedSize();
diskUsage.setSizeWithReplica(keyDU);
}

if (listFile) {
duData.add(diskUsage);
}

return diskUsage;
}


/**
* Given a valid path request for a directory,
Expand Down

0 comments on commit c86182e

Please sign in to comment.