Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support adding columns to multiple rollup indexes in one ALTER TABLE stmt #931

Merged
merged 3 commits into from
Apr 16, 2019
Merged

Support adding columns to multiple rollup indexes in one ALTER TABLE stmt #931

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
98b1e82
Support add columns to multiple rollup indexes in one ALTER operation
morningman Apr 16, 2019
8de4783
fix bug
morningman Apr 16, 2019
9e7c076
change way
morningman Apr 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/alter/AlterHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ protected void unlock() {
}

public AlterHandler(String name) {
super(name, 20000);
super(name, 10000);
}

protected void addAlterJob(AlterJob alterJob) {
Expand Down
55 changes: 35 additions & 20 deletions fe/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,9 @@ private void processAddColumn(AddColumnClause alterClause, OlapTable olapTable,
targetIndexId = olapTable.getIndexIdByName(targetIndexName);
}

Set<String> newColNameSet = Sets.newHashSet(column.getName());
addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId,
baseIndexName, indexSchemaMap);
baseIndexName, indexSchemaMap, newColNameSet);
}

private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable,
Expand All @@ -113,10 +114,12 @@ private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);

Set<String> newColNameSet = Sets.newHashSet();
for (Column column : columns) {
if (column.isKey()) {
checkKeyModificationIfInRandomDistributedTable(olapTable);
}
newColNameSet.add(column.getName());
}

String baseIndexName = olapTable.getName();
Expand All @@ -128,9 +131,9 @@ private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable
targetIndexId = olapTable.getIndexIdByName(targetIndexName);
}

for (Column columnDef : columns) {
addColumnInternal(olapTable, columnDef, null, targetIndexId, baseIndexId,
baseIndexName, indexSchemaMap);
for (Column column : columns) {
addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId,
baseIndexName, indexSchemaMap, newColNameSet);
}
}

Expand Down Expand Up @@ -488,7 +491,8 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol

private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos,
long targetIndexId, long baseIndexId, String baseIndexName,
Map<Long, LinkedList<Column>> indexSchemaMap) throws DdlException {
Map<Long, LinkedList<Column>> indexSchemaMap,
Set<String> newColNameSet) throws DdlException {

if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
if (newColumn.isKey() && newColumn.getAggregationType() != null) {
Expand Down Expand Up @@ -533,59 +537,58 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
// check if has default value. this should be done in Analyze phase
// 1. add to base index first
// List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
// checkAndAddColumn(modIndexSchema, newColumn, columnPos);
List<Column> modIndexSchema;
if (newColumn.isKey()) {
// add key column to unique key table, should add to all rollups
// Column column = olapTable.getColumn(columnPos.getLastCol());
// add to all table including base and rollup
for (Map.Entry<Long, LinkedList<Column>> entry : indexSchemaMap.entrySet()) {
modIndexSchema = entry.getValue();
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
boolean isBaseIdex = entry.getKey() == baseIndexId;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, isBaseIdex);
}
} else {
// 1. add to base table
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
if (targetIndexId == -1L) {
return;
}

// 2. add to rollup
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
}
} else if (KeysType.DUP_KEYS == olapTable.getKeysType()) {
if (targetIndexId == -1L) {
// check if has default value. this should be done in Analyze phase
// 1. add to base index first
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
// no specified target index. return
return;
} else {
// 2. add to rollup index
List<Column> modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);

if (newColumn.isKey()) {
/*
* if add column in rollup is key,
* then put the column in base table as end key
*/
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, null);
checkAndAddColumn(modIndexSchema, newColumn, null, newColNameSet, true);
} else {
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
}
}
} else {
// check if has default value. this should be done in Analyze phase
// 1. add to base index first
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);

if (targetIndexId == -1L) {
// no specified target index. return
Expand All @@ -594,21 +597,33 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi

// 2. add to rollup index
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
}
}

private void checkAndAddColumn(List<Column> modIndexSchema, Column newColumn, ColumnPosition columnPos)
throws DdlException {
private void checkAndAddColumn(List<Column> modIndexSchema, Column newColumn, ColumnPosition columnPos,
Set<String> newColNameSet, boolean isBaseIndex) throws DdlException {
int posIndex = -1;
String newColName = newColumn.getName();
boolean hasPos = (columnPos != null && !columnPos.isFirst());
for (int i = 0; i < modIndexSchema.size(); i++) {
Column col = modIndexSchema.get(i);
if (col.getName().equalsIgnoreCase(newColName)) {
// already add
throw new DdlException("Duplicately add column[" + newColName + "]");
if (!isBaseIndex || !newColNameSet.contains(newColName)) {
// if this is not a base index, we should check if user repeatedly add columns
throw new DdlException("Repeatedly add column[" + newColName + "]");
}
// this is a base index, and the column we check here is added by previous 'add column clause'
// in same ALTER stmt.
// so here we will check if the 2 columns is exactly same. if not, throw exception
if (!col.equals(newColumn)) {
throw new DdlException("Repeatedly add same column[" + newColName + "] with different definition");
}

// column already exist, return
return;
}

if (hasPos) {
// after the field
if (col.getName().equalsIgnoreCase(columnPos.getLastCol())) {
Expand Down
4 changes: 2 additions & 2 deletions fe/src/main/java/org/apache/doris/analysis/AddColumnClause.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@

import com.google.common.base.Strings;

import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Map;

// clause which is used to add one column to
public class AddColumnClause extends AlterClause {
private static final Logger LOG = LogManager.getLogger(AddColumnClause.class);
Expand Down
5 changes: 3 additions & 2 deletions fe/src/main/java/org/apache/doris/analysis/AddColumnsClause.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@

package org.apache.doris.analysis;

import com.google.common.collect.Lists;
import org.apache.doris.catalog.Column;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;

import com.google.common.base.Strings;
import com.google.common.collect.Lists;

import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -69,7 +69,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException {

columns = Lists.newArrayList();
for (ColumnDef columnDef : columnDefs) {
columns.add(columnDef.toColumn());
Column col = columnDef.toColumn();
columns.add(col);
}
}

Expand Down
5 changes: 5 additions & 0 deletions fe/src/main/java/org/apache/doris/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,11 @@ public boolean equals(Object obj) {
if (this.getScale() != other.getScale()) {
return false;
}

if (!comment.equals(other.getComment())) {
return false;
}

return true;
}

Expand Down