Skip to content

Commit

Permalink
HBASE-27938 - Add ut to capture to validate parse options for custom …
Browse files Browse the repository at this point in the history 
…test class and properties
  • Loading branch information
Prathyusha Garre committed May 10, 2024
1 parent 753391a commit 29af40a
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 13 deletions.
36 changes: 23 additions & 13 deletions hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,11 +360,14 @@ static boolean checkTable(Admin admin, TestOptions opts) throws IOException {
// recreate the table when user has requested presplit or when existing
// {RegionSplitPolicy,replica count} does not match requested, or when the
// number of column families does not match requested.
if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions
if (
(exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions
&& opts.presplitRegions != admin.getRegions(tableName).size())
|| (!isReadCmd && desc != null && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy))
|| (!isReadCmd && desc != null
&& !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy))
|| (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)
|| (desc != null && desc.getColumnFamilyCount() != opts.families)) {
|| (desc != null && desc.getColumnFamilyCount() != opts.families)
) {
needsDelete = true;
// wait, why did it delete my table?!?
LOG.debug(MoreObjects.toStringHelper("needsDelete").add("needsDelete", needsDelete)
Expand Down Expand Up @@ -786,11 +789,11 @@ public TestOptions(TestOptions that) {
this.bufferSize = that.bufferSize;
this.commandProperties = that.commandProperties;
}

public Properties getCommandProperties() {
return commandProperties;
}

public int getCaching() {
return this.caching;
}
Expand Down Expand Up @@ -2313,7 +2316,7 @@ protected byte[] generateRow(final int i) {
}

@Override
protected boolean testRow(final int i, final long startTime) throws IOException {
boolean testRow(final int i, final long startTime) throws IOException {
byte[] row = generateRow(i);
Put put = new Put(row);
for (int family = 0; family < opts.families; family++) {
Expand Down Expand Up @@ -2586,7 +2589,7 @@ protected static void printUsage(final String shortName, final String message) {
System.err.println(message);
}
System.err.print("Usage: hbase " + shortName);
System.err.println(" <OPTIONS> [-D<property=value>]* <command> <nclients>");
System.err.println(" <OPTIONS> [-D<property=value>]* <command|class> <nclients>");
System.err.println();
System.err.println("General Options:");
System.err.println(
Expand Down Expand Up @@ -2686,7 +2689,12 @@ protected static void printUsage(final String shortName, final String message) {
for (CmdDescriptor command : COMMANDS.values()) {
System.err.println(String.format(" %-20s %s", command.getName(), command.getDescription()));
}
System.err.println(String.format(" %-20s %s", "? extends Test", "Run custom implementation of provided Test present in classpath"));
System.err.println();
System.err.println("Class:");
System.err.println(
"To run any custom implementation of PerformanceEvaluation.Test, provide the classname of the implementaion class in place of command name and it will be loaded at runtime from classpath.:");
System.err.println(
"Please consider to contribute back this custom test impl into a builtin PE command for the benefit of the community");
System.err.println();
System.err.println("Args:");
System.err.println(" nclients Integer. Required. Total number of clients "
Expand Down Expand Up @@ -2987,14 +2995,15 @@ static TestOptions parseOpts(Queue<String> args) {
String fileName = String.valueOf(cmd.substring(commandPropertiesFile.length()));
Properties properties = new Properties();
try {
properties.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName));
properties
.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName));
opts.commandProperties = properties;
} catch (IOException e) {
LOG.error("Failed to load metricIds from properties file", e);
}
continue;
}

validateParsedOpts(opts);

if (isCommandClass(cmd)) {
Expand Down Expand Up @@ -3108,21 +3117,22 @@ public int run(String[] args) throws Exception {
}

private static boolean isCommandClass(String cmd) {
return !COMMANDS.containsKey(cmd) ? isCustomTestClass(cmd) : true;
return COMMANDS.containsKey(cmd) || isCustomTestClass(cmd);
}

private static boolean isCustomTestClass(String cmd) {
Class<? extends Test> cmdClass;
try {
cmdClass = (Class<? extends Test>) PerformanceEvaluation.class.getClassLoader().loadClass(cmd);
cmdClass =
(Class<? extends Test>) PerformanceEvaluation.class.getClassLoader().loadClass(cmd);
addCommandDescriptor(cmdClass, cmd, "custom command");
return true;
} catch (Throwable th) {
LOG.info("No class found for command: " + cmd, th);
return false;
}
}

private static Class<? extends TestBase> determineCommandClass(String cmd) {
CmdDescriptor descriptor = COMMANDS.get(cmd);
return descriptor != null ? descriptor.getCmdClass() : null;
Expand Down
47 changes: 47 additions & 0 deletions hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,27 @@
import com.codahale.metrics.UniformReservoir;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.PerformanceEvaluation.RandomReadTest;
import org.apache.hadoop.hbase.PerformanceEvaluation.Status;
import org.apache.hadoop.hbase.PerformanceEvaluation.TestOptions;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
import org.apache.hadoop.hbase.testclassification.MiscTests;
Expand Down Expand Up @@ -359,4 +364,46 @@ public void testParseOptsValueRandom() {
assertEquals(true, options.valueRandom);
}

@Test
public void testCustomTestClassOptions() throws IOException {
Queue<String> opts = new LinkedList<>();
// create custom properties that can be used for a custom test class
Properties commandProps = new Properties();
commandProps.put("prop1", "val1");
String cmdPropsFilePath =
this.getClass().getClassLoader().getResource("").getPath() + "cmd_properties.txt";
FileWriter writer = new FileWriter(new File(cmdPropsFilePath));
commandProps.store(writer, null);
// create opts for the custom test class - commandPropertiesFile, testClassName
opts.offer("--commandPropertiesFile=" + "cmd_properties.txt");
String testClassName = "org.apache.hadoop.hbase.TestPerformanceEvaluation$PESampleTestImpl";
opts.offer(testClassName);
opts.offer("1");
PerformanceEvaluation.TestOptions options = PerformanceEvaluation.parseOpts(opts);
assertNotNull(options);
assertNotNull(options.getCmdName());
assertEquals(testClassName, options.getCmdName());
assertNotNull(options.getCommandProperties());
assertEquals("val1", options.getCommandProperties().get("prop1"));
}

class PESampleTestImpl extends PerformanceEvaluation.Test {

PESampleTestImpl(Connection con, TestOptions options, Status status) {
super(con, options, status);
}

@Override
void onStartup() throws IOException {
}

@Override
void onTakedown() throws IOException {
}

@Override
boolean testRow(int i, long startTime) throws IOException, InterruptedException {
return false;
}
}
}

0 comments on commit 29af40a

Please sign in to comment.