Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support create index on unique value column #5305

Merged
merged 4 commits into from
Feb 3, 2021
Merged

Support create index on unique value column #5305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
27feb0a
support create index on unique table value columns
yangzhg Jan 27, 2021
3be1b9e
support create index on unique table value columns
yangzhg Jan 27, 2021
dc35643
support create index on unique table value columns
yangzhg Jan 27, 2021
63c9bac
support create index on unique table value columns
yangzhg Jan 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions be/src/olap/rowset/segment_v2/segment_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,9 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec __attribute__((unused))

// now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS
// and not support zone map for array type.
opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() == KeysType::DUP_KEYS;
opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS;
if (column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
opts.need_zone_map = false;
} else {
opts.need_zone_map =
column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS;
}
opts.need_bloom_filter = column.is_bf_column();
opts.need_bitmap_index = column.has_bitmap_index();
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1032,7 +1032,8 @@ private void createJob(long dbId, OlapTable olapTable, Map<Long, LinkedList<Colu
Set<String> bfColumns = null;
double bfFpp = 0;
try {
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap, indexSchemaMap.get(olapTable.getBaseIndexId()));
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap,
indexSchemaMap.get(olapTable.getBaseIndexId()), olapTable.getKeysType());
bfFpp = PropertyAnalyzer.analyzeBloomFilterFpp(propertyMap);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ public String getComment() {

public enum IndexType {
BITMAP,

}

public void checkColumn(Column column, KeysType keysType) throws AnalysisException {
Expand All @@ -130,11 +131,10 @@ public void checkColumn(Column column, KeysType keysType) throws AnalysisExcepti
colType.isStringType() || colType == PrimitiveType.BOOLEAN)) {
throw new AnalysisException(colType + " is not supported in bitmap index. "
+ "invalid column: " + indexColName);
} else if (((keysType == KeysType.AGG_KEYS || keysType == KeysType.UNIQUE_KEYS) && !column.isKey())
|| keysType == KeysType.PRIMARY_KEYS) {
} else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) {
throw new AnalysisException(
"BITMAP index only used in columns of DUP_KEYS table or key columns of"
+ " UNIQUE_KEYS/AGG_KEYS table. invalid column: " + indexColName);
"BITMAP index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
}
} else {
throw new AnalysisException("Unsupported index type: " + indexType);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3637,7 +3637,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws DdlExcept
Set<String> bfColumns = null;
double bfFpp = 0;
try {
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema);
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema, keysType);
if (bfColumns != null && bfColumns.isEmpty()) {
bfColumns = null;
}
Expand Down
15 changes: 6 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.doris.common.util;

import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.KeysType;
Expand Down Expand Up @@ -299,8 +298,8 @@ public static int analyzeSchemaVersion(Map<String, String> properties) throws An
return schemaVersion;
}

public static Set<String> analyzeBloomFilterColumns(Map<String, String> properties, List<Column> columns)
throws AnalysisException {
public static Set<String> analyzeBloomFilterColumns(Map<String, String> properties, List<Column> columns,
KeysType keysType) throws AnalysisException {
Set<String> bfColumns = null;
if (properties != null && properties.containsKey(PROPERTIES_BF_COLUMNS)) {
bfColumns = Sets.newHashSet();
Expand All @@ -324,8 +323,7 @@ public static Set<String> analyzeBloomFilterColumns(Map<String, String> properti
|| type == PrimitiveType.DOUBLE || type == PrimitiveType.BOOLEAN) {
throw new AnalysisException(type + " is not supported in bloom filter index. "
+ "invalid column: " + bfColumn);
} else if (column.isKey()
|| column.getAggregationType() == AggregateType.NONE) {
} else if (keysType != KeysType.AGG_KEYS || column.isKey()) {
if (!bfColumnSet.add(bfColumn)) {
throw new AnalysisException("Reduplicated bloom filter column: " + bfColumn);
}
Expand All @@ -334,10 +332,9 @@ public static Set<String> analyzeBloomFilterColumns(Map<String, String> properti
found = true;
break;
} else {
// althrough the implemention supports bf for replace non-key column,
// for simplicity and unity, we don't expose that to user.
throw new AnalysisException("Bloom filter index only used in columns of DUP_KEYS table or "
+ "key columns of UNIQUE_KEYS/AGG_KEYS table. invalid column: " + bfColumn);
throw new AnalysisException("Bloom filter index only used in columns of" +
" UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." +
" invalid column: " + bfColumn);
}
}
}
Expand Down
16 changes: 9 additions & 7 deletions fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
Expand Down Expand Up @@ -63,7 +64,7 @@ public void testBfColumns() throws AnalysisException {
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1");

Set<String> bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
Set<String> bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
Assert.assertEquals(Sets.newHashSet("k1"), bfColumns);
}

Expand All @@ -84,47 +85,48 @@ public void testBfColumnsError() {
// no bf columns
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "");
try {
Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns));
Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns,
KeysType.AGG_KEYS));
} catch (AnalysisException e) {
Assert.fail();
}

// k4 not exist
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k4");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("column does not exist in table"));
}

// tinyint not supported
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k2");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("TINYINT is not supported"));
}

// bool not supported
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k3");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("BOOLEAN is not supported"));
}

// not replace value
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "v2");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in"));
}

// reduplicated column
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1,K1");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("Reduplicated bloom filter column"));
}
Expand Down