Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Custom config handler #6508 #6577

Merged
merged 1 commit into from
Nov 7, 2021
Merged

Custom config handler #6508 #6577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 89 additions & 115 deletions fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.common;

import org.apache.doris.catalog.Catalog;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -39,21 +40,36 @@
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class ConfigBase {
private static final Logger LOG = LogManager.getLogger(ConfigBase.class);

@Retention(RetentionPolicy.RUNTIME)
public static @interface ConfField {
public @interface ConfField {
String value() default "";
boolean mutable() default false;
boolean masterOnly() default false;
String comment() default "";
Class<? extends ConfHandler> callback() default DefaultConfHandler.class;
}

public interface ConfHandler {
void handle(Field field, String confVal) throws Exception;
}

static class DefaultConfHandler implements ConfHandler {
@Override
public void handle(Field field, String confVal) throws Exception{
setConfigField(field, confVal);
}
}

private static String confFile;
private static String customConfFile;
public static Class<? extends ConfigBase> confClass;
public static Map<String, Field> confFields;

private static String ldapConfFile;
private static String ldapCustomConfFile;
Expand All @@ -66,6 +82,15 @@ public void init(String configFile) throws Exception {
if (!isLdapConfig) {
confClass = this.getClass();
confFile = configFile;
confFields = Maps.newHashMap();
for (Field field : confClass.getFields()) {
ConfField confField = field.getAnnotation(ConfField.class);
if (confField == null) {
continue;
}
confFields.put(confField.value().equals("") ? field.getName() : confField.value(), field);
}

initConf(confFile);
} else {
ldapConfClass = this.getClass();
Expand All @@ -90,42 +115,48 @@ private void initConf(String confFile) throws Exception {
replacedByEnv(props);
setFields(props, isLdapConfig);
}
public static HashMap<String, String> dump() throws Exception {
HashMap<String, String> map = new HashMap<String, String>();
Field[] fields = confClass.getFields();

public static HashMap<String, String> dump() {
HashMap<String, String> map = new HashMap<>();
Field[] fields = confClass.getFields();
for (Field f : fields) {
if (f.getAnnotation(ConfField.class) == null) {
continue;
ConfField anno = f.getAnnotation(ConfField.class);
if (anno != null) {
map.put(anno.value().isEmpty() ? f.getName() : anno.value(), getConfValue(f));
}
if (f.getType().isArray()) {
switch (f.getType().getSimpleName()) {
}
return map;
}

public static String getConfValue(Field field) {
try {
if (field.getType().isArray()) {
switch (field.getType().getSimpleName()) {
case "boolean[]":
return Arrays.toString((boolean[]) field.get(null));
case "char[]":
return Arrays.toString((char[]) field.get(null));
case "byte[]":
return Arrays.toString((byte[]) field.get(null));
case "short[]":
map.put(f.getName(), Arrays.toString((short[]) f.get(null)));
break;
return Arrays.toString((short[]) field.get(null));
case "int[]":
map.put(f.getName(), Arrays.toString((int[]) f.get(null)));
break;
return Arrays.toString((int[]) field.get(null));
case "long[]":
map.put(f.getName(), Arrays.toString((long[]) f.get(null)));
break;
return Arrays.toString((long[]) field.get(null));
case "float[]":
return Arrays.toString((float[]) field.get(null));
case "double[]":
map.put(f.getName(), Arrays.toString((double[]) f.get(null)));
break;
case "boolean[]":
map.put(f.getName(), Arrays.toString((boolean[]) f.get(null)));
break;
case "String[]":
map.put(f.getName(), Arrays.toString((String[]) f.get(null)));
break;
return Arrays.toString((double[]) field.get(null));
default:
throw new Exception("unknown type: " + f.getType().getSimpleName());
}
return Arrays.toString((Object[]) field.get(null));
}
} else {
map.put(f.getName(), f.get(null).toString());
return String.valueOf(field.get(null));
}
} catch (Exception e) {
return String.format("Failed to get config %s: %s", field.getName(), e.getMessage());
}
return map;
}

// there is some config in fe.conf like:
Expand Down Expand Up @@ -176,7 +207,7 @@ private static void setFields(Properties props, boolean isLdapConfig) throws Exc
}
}

public static void setConfigField(Field f, String confVal) throws IllegalAccessException, Exception {
public static void setConfigField(Field f, String confVal) throws Exception {
confVal = confVal.trim();

String[] sa = confVal.split(",");
Expand Down Expand Up @@ -258,114 +289,57 @@ private static boolean isBoolean(String s) {
throw new IllegalArgumentException("type mismatch");
}

public static Map<String, Field> getAllMutableConfigs() {
Map<String, Field> mutableConfigs = Maps.newHashMap();
Field fields[] = ConfigBase.confClass.getFields();
for (Field field : fields) {
ConfField confField = field.getAnnotation(ConfField.class);
if (confField == null) {
continue;
}
if (!confField.mutable()) {
continue;
}
mutableConfigs.put(confField.value().equals("") ? field.getName() : confField.value(), field);
}

return mutableConfigs;
}

public synchronized static void setMutableConfig(String key, String value) throws DdlException {
Map<String, Field> mutableConfigs = getAllMutableConfigs();
Field field = mutableConfigs.get(key);
Field field = confFields.get(key);
if (field == null) {
throw new DdlException("Config '" + key + "' does not exist or is not mutable");
throw new DdlException("Config '" + key + "' does not exist");
}

ConfField anno = field.getAnnotation(ConfField.class);
if (!anno.mutable()) {
throw new DdlException("Config '" + key + "' is not mutable");
}
if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()){
throw new DdlException("Config '" + key + "' is master only");
}

try {
ConfigBase.setConfigField(field, value);
anno.callback().newInstance().handle(field, value);
} catch (Exception e) {
throw new DdlException("Failed to set config '" + key + "'. err: " + e.getMessage());
}

LOG.info("set config {} to {}", key, value);
}

public synchronized static List<List<String>> getConfigInfo(PatternMatcher matcher) throws DdlException {
List<List<String>> configs = Lists.newArrayList();
Field[] fields = confClass.getFields();
for (Field f : fields) {
List<String> config = Lists.newArrayList();
public synchronized static List<List<String>> getConfigInfo(PatternMatcher matcher) {
return confFields.entrySet().stream().sorted(Map.Entry.comparingByKey()).flatMap(e -> {
String confKey = e.getKey();
Field f = e.getValue();
ConfField anno = f.getAnnotation(ConfField.class);
if (anno == null) {
continue;
}

String confKey = anno.value().equals("") ? f.getName() : anno.value();
if (matcher != null && !matcher.match(confKey)) {
continue;
}
String confVal;
try {
switch (f.getType().getSimpleName()) {
case "short":
case "int":
case "long":
case "double":
case "boolean":
case "String":
confVal = String.valueOf(f.get(null));
break;
case "short[]":
confVal = Arrays.toString((short[])f.get(null));
break;
case "int[]":
confVal = Arrays.toString((int[])f.get(null));
break;
case "long[]":
confVal = Arrays.toString((long[])f.get(null));
break;
case "double[]":
confVal = Arrays.toString((double[])f.get(null));
break;
case "boolean[]":
confVal = Arrays.toString((boolean[])f.get(null));
break;
case "String[]":
confVal = Arrays.toString((String[])f.get(null));
break;
default:
throw new DdlException("unknown type: " + f.getType().getSimpleName());
}
} catch (IllegalArgumentException | IllegalAccessException e) {
throw new DdlException("Failed to get config '" + confKey + "'. err: " + e.getMessage());
if (matcher == null || matcher.match(confKey)) {
List<String> config = Lists.newArrayList();
config.add(confKey);
config.add(getConfValue(f));
config.add(f.getType().getSimpleName());
config.add(String.valueOf(anno.mutable()));
config.add(String.valueOf(anno.masterOnly()));
config.add(anno.comment());
return Stream.of(config);
} else {
return Stream.empty();
}

config.add(confKey);
config.add(Strings.nullToEmpty(confVal));
config.add(f.getType().getSimpleName());
config.add(String.valueOf(anno.mutable()));
config.add(String.valueOf(anno.masterOnly()));
config.add(anno.comment());
configs.add(config);
}

return configs;
}).collect(Collectors.toList());
}

public synchronized static boolean checkIsMasterOnly(String key) {
Map<String, Field> mutableConfigs = getAllMutableConfigs();
Field f = mutableConfigs.get(key);
Field f = confFields.get(key);
if (f == null) {
return false;
}

ConfField anno = f.getAnnotation(ConfField.class);
if (anno == null) {
return false;
}

return anno.masterOnly();
return anno != null && anno.mutable() && anno.masterOnly();
}

// use synchronized to make sure only one thread modify this file
Expand Down
50 changes: 12 additions & 38 deletions fe/fe-core/src/main/java/org/apache/doris/http/rest/SetConfigAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@

import io.netty.handler.codec.http.HttpMethod;

import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ConfigBase.ConfField;
import org.apache.doris.common.DdlException;
import org.apache.doris.http.ActionController;
import org.apache.doris.http.BaseRequest;
Expand All @@ -37,7 +35,6 @@
import org.codehaus.jackson.map.ObjectMapper;

import java.io.IOException;
import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -86,37 +83,20 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response

LOG.debug("get config from url: {}, need persist: {}", configs, needPersist);

Field[] fields = ConfigBase.confClass.getFields();
for (Field f : fields) {
// ensure that field has "@ConfField" annotation
ConfField anno = f.getAnnotation(ConfField.class);
if (anno == null || !anno.mutable()) {
continue;
}

if (anno.masterOnly() && !Catalog.getCurrentCatalog().isMaster()) {
continue;
}

// ensure that field has property string
String confKey = anno.value().equals("") ? f.getName() : anno.value();
List<String> confVals = configs.get(confKey);
if (confVals == null || confVals.isEmpty()) {
continue;
}

if (confVals.size() > 1) {
continue;
}

for (Map.Entry<String, List<String>> config : configs.entrySet()) {
String confKey = config.getKey();
List<String> confValue = config.getValue();
try {
ConfigBase.setConfigField(f, confVals.get(0));
} catch (Exception e) {
LOG.warn("failed to set config {}:{}", confKey, confVals.get(0), e);
continue;
if (confValue != null && confValue.size() == 1) {
ConfigBase.setMutableConfig(confKey, confValue.get(0));
setConfigs.put(confKey, confValue.get(0));
} else {
throw new DdlException("conf value size != 1");
}
} catch (DdlException e) {
LOG.warn("failed to set config {}:{}", confKey, confValue, e);
errConfigs.put(confKey, String.valueOf(confValue));
}

setConfigs.put(confKey, confVals.get(0));
}

String persistMsg = "";
Expand All @@ -130,12 +110,6 @@ protected void executeWithoutPassword(BaseRequest request, BaseResponse response
}
}

for (String key : configs.keySet()) {
if (!setConfigs.containsKey(key)) {
errConfigs.put(key, configs.get(key).toString());
}
}

Map<String, Object> resultMap = Maps.newHashMap();
resultMap.put("set", setConfigs);
resultMap.put("err", errConfigs);
Expand Down
Loading