[fix](build index)Fix build index failed on renamed column (#42882)(#43246)(#43336)

be/src/olap/tablet_schema.cpp

@@ -739,7 +739,15 @@ void TabletIndex::init_from_thrift(const TOlapTableIndex& index,
         if (column_idx >= 0) {
             col_unique_ids[i] = tablet_schema.column(column_idx).unique_id();
         } else {
-            col_unique_ids[i] = -1;
+            // if column unique id not found by column name, find by column unique id
+            // column unique id can not bigger than tablet schema column size, if bigger than column size means
+            // this column is a new column added by light schema change
+            if (index.__isset.column_unique_ids &&
+                index.column_unique_ids[i] < tablet_schema.num_columns()) {
+                col_unique_ids[i] = index.column_unique_ids[i];
+            } else {
+                col_unique_ids[i] = -1;
+            }
         }
     }
     _col_unique_ids = std::move(col_unique_ids);

be/src/olap/task/index_builder.cpp

@@ -83,10 +83,16 @@ Status IndexBuilder::update_inverted_index_info() {
                 auto column_name = t_inverted_index.columns[0];
                 auto column_idx = output_rs_tablet_schema->field_index(column_name);
                 if (column_idx < 0) {
-                    LOG(WARNING) << "referenced column was missing. "
-                                 << "[column=" << column_name << " referenced_column=" << column_idx
-                                 << "]";
-                    continue;
+                    if (!t_inverted_index.column_unique_ids.empty()) {
+                        auto column_unique_id = t_inverted_index.column_unique_ids[0];
+                        column_idx = output_rs_tablet_schema->field_index(column_unique_id);
+                    }
+                    if (column_idx < 0) {
+                        LOG(WARNING) << "referenced column was missing. "
+                                     << "[column=" << column_name
+                                     << " referenced_column=" << column_idx << "]";
+                        continue;
+                    }
                 }
                 auto column = output_rs_tablet_schema->column(column_idx);
                 const auto* index_meta = output_rs_tablet_schema->get_inverted_index(column);
@@ -346,10 +352,16 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
                 auto column_name = inverted_index.columns[0];
                 auto column_idx = output_rowset_schema->field_index(column_name);
                 if (column_idx < 0) {
-                    LOG(WARNING) << "referenced column was missing. "
-                                 << "[column=" << column_name << " referenced_column=" << column_idx
-                                 << "]";
-                    continue;
+                    if (!inverted_index.column_unique_ids.empty()) {
+                        column_idx = output_rowset_schema->field_index(
+                                inverted_index.column_unique_ids[0]);
+                    }
+                    if (column_idx < 0) {
+                        LOG(WARNING) << "referenced column was missing. "
+                                     << "[column=" << column_name
+                                     << " referenced_column=" << column_idx << "]";
+                        continue;
+                    }
                 }
                 auto column = output_rowset_schema->column(column_idx);
                 if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) {
@@ -478,9 +490,16 @@ Status IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema,
         auto column_name = inverted_index.columns[0];
         auto column_idx = tablet_schema->field_index(column_name);
         if (column_idx < 0) {
-            LOG(WARNING) << "referenced column was missing. "
-                         << "[column=" << column_name << " referenced_column=" << column_idx << "]";
-            continue;
+            if (!inverted_index.column_unique_ids.empty()) {
+                auto column_unique_id = inverted_index.column_unique_ids[0];
+                column_idx = tablet_schema->field_index(column_unique_id);
+            }
+            if (column_idx < 0) {
+                LOG(WARNING) << "referenced column was missing. "
+                             << "[column=" << column_name << " referenced_column=" << column_idx
+                             << "]";
+                continue;
+            }
         }
         auto column = tablet_schema->column(column_idx);
         auto writer_sign = std::make_pair(segment_idx, index_id);

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java

@@ -2012,6 +2012,7 @@ public int getAsInt() {
                             index.setIndexId(existedIdx.getIndexId());
                             index.setColumns(existedIdx.getColumns());
                             index.setProperties(existedIdx.getProperties());
+                            index.setColumnUniqueIds(existedIdx.getColumnUniqueIds());
                             if (indexDef.getPartitionNames().isEmpty()) {
                                 invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames());
                             } else {
@@ -2588,6 +2589,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
             if (column != null) {
                 indexDef.checkColumn(column, olapTable.getKeysType(),
                         olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite());
+                indexDef.getColumnUniqueIds().add(column.getUniqueId());
             } else {
                 throw new DdlException("index column does not exist in table. invalid column: " + col);
             }
@@ -2598,6 +2600,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
         // so here update column name in CreateIndexClause after checkColumn for indexDef,
         // there will use the column name in olapTable insead of the column name in CreateIndexClause.
         alterIndex.setColumns(indexDef.getColumns());
+        alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds());
         newIndexes.add(alterIndex);
         return false;
     }

fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java

@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         indexDef.analyze();
         this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
                 indexDef.getColumns(), indexDef.getIndexType(),
-                indexDef.getProperties(), indexDef.getComment());
+                indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java

@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
         indexDef.analyze();
         this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
                 indexDef.getColumns(), indexDef.getIndexType(),
-                indexDef.getProperties(), indexDef.getComment());
+                indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java

@@ -581,7 +581,8 @@ public void analyze(Analyzer analyzer) throws UserException {
                     }
                 }
                 indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
-                        indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
+                        indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(),
+                        indexDef.getColumnUniqueIds()));
                 distinct.add(indexDef.getIndexName());
                 distinctCol.add(Pair.of(indexDef.getIndexType(),
                         indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));

fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java

@@ -43,6 +43,7 @@ public class IndexDef {
     private Map<String, String> properties;
     private boolean isBuildDeferred = false;
     private PartitionNames partitionNames;
+    private List<Integer> columnUniqueIds = Lists.newArrayList();
 
     public static final String NGRAM_SIZE_KEY = "gram_size";
     public static final String NGRAM_BF_SIZE_KEY = "bf_size";
@@ -197,6 +198,10 @@ public List<String> getPartitionNames() {
         return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames();
     }
 
+    public List<Integer> getColumnUniqueIds() {
+        return columnUniqueIds;
+    }
+
     public enum IndexType {
         BITMAP,
         INVERTED,

fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java

@@ -47,7 +47,7 @@
 
 /**
  * Internal representation of index, including index type, name, columns and comments.
- * This class will used in olaptable
+ * This class will be used in olap table
  */
 public class Index implements Writable {
     public static final int INDEX_ID_INIT_VALUE = -1;
@@ -64,15 +64,19 @@ public class Index implements Writable {
     private Map<String, String> properties;
     @SerializedName(value = "comment")
     private String comment;
+    @SerializedName(value = "cui", alternate = {"columnUniqueIds"})
+    private List<Integer> columnUniqueIds;
 
     public Index(long indexId, String indexName, List<String> columns,
-            IndexDef.IndexType indexType, Map<String, String> properties, String comment) {
+            IndexDef.IndexType indexType, Map<String, String> properties, String comment,
+            List<Integer> columnUniqueIds) {
         this.indexId = indexId;
         this.indexName = indexName;
         this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns);
         this.indexType = indexType;
         this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties);
         this.comment = comment;
+        this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds);
         if (indexType == IndexDef.IndexType.INVERTED) {
             if (this.properties != null && !this.properties.isEmpty()) {
                 if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
@@ -96,6 +100,7 @@ public Index() {
         this.indexType = null;
         this.properties = null;
         this.comment = null;
+        this.columnUniqueIds = null;
     }
 
     public long getIndexId() {
@@ -185,6 +190,14 @@ public void setComment(String comment) {
         this.comment = comment;
     }
 
+    public List<Integer> getColumnUniqueIds() {
+        return columnUniqueIds;
+    }
+
+    public void setColumnUniqueIds(List<Integer> columnUniqueIds) {
+        this.columnUniqueIds = columnUniqueIds;
+    }
+
     @Override
     public void write(DataOutput out) throws IOException {
         Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -202,7 +215,7 @@ public int hashCode() {
 
     public Index clone() {
         return new Index(indexId, indexName, new ArrayList<>(columns),
-                indexType, new HashMap<>(properties), comment);
+                indexType, new HashMap<>(properties), comment, columnUniqueIds);
     }
 
     @Override
@@ -246,6 +259,7 @@ public TOlapTableIndex toThrift() {
         if (properties != null) {
             tIndex.setProperties(properties);
         }
+        tIndex.setColumnUniqueIds(columnUniqueIds);
         return tIndex;
     }
 

fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java

@@ -387,6 +387,13 @@ public void initSchemaColumnUniqueId() {
         maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
         this.schema.forEach(column -> {
             column.setUniqueId(incAndGetMaxColUniqueId());
+            this.indexes.forEach(index -> {
+                index.getColumns().forEach(col -> {
+                    if (col.equalsIgnoreCase(column.getName())) {
+                        index.getColumnUniqueIds().add(column.getUniqueId());
+                    }
+                });
+            });
             if (LOG.isDebugEnabled()) {
                 LOG.debug("indexId: {},  column:{}, uniqueId:{}",
                         indexId, column, column.getUniqueId());

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java

@@ -211,6 +211,6 @@ public IndexType getIndexType() {
 
     public Index translateToCatalogStyle() {
         return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties,
-                comment);
+                comment, null);
     }
 }

fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java

@@ -58,7 +58,7 @@ int getCurrentEnvJournalVersion() {
             }
             OlapTable tbl = (OlapTable) table;
             tbl.setIndexes(Lists.newArrayList(new Index(0, "index", Lists.newArrayList("col"),
-                    IndexDef.IndexType.BITMAP, null, "xxxxxx")));
+                    IndexDef.IndexType.BITMAP, null, "xxxxxx", Lists.newArrayList(1))));
             System.out.println("orig table id: " + tbl.getId());
 
             FastByteArrayOutputStream byteArrayOutputStream = new FastByteArrayOutputStream();

fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java

@@ -42,18 +42,18 @@ public class IndexesProcNodeTest {
     public void testFetchResult() throws AnalysisException {
         List<Index> indexes = new ArrayList<>();
         Index indexBitmap = new Index(1, "bitmap_index", Lists.newArrayList("col_1"),
-                IndexType.BITMAP, null, "bitmap index on col_1");
+                IndexType.BITMAP, null, "bitmap index on col_1", Lists.newArrayList(1));
         Map<String, String> invertedProperties = new HashMap<>();
         invertedProperties.put("parser", "unicode");
         Index indexInverted = new Index(2, "inverted_index", Lists.newArrayList("col_2"),
-                        IndexType.INVERTED, invertedProperties, "inverted index on col_2");
+                        IndexType.INVERTED, invertedProperties, "inverted index on col_2", Lists.newArrayList(2));
         Index indexBf = new Index(3, "bloomfilter_index", Lists.newArrayList("col_3"),
-                IndexType.BLOOMFILTER, null, "bloomfilter index on col_3");
+                IndexType.BLOOMFILTER, null, "bloomfilter index on col_3", Lists.newArrayList(3));
         Map<String, String> ngramProperties = new HashMap<>();
         ngramProperties.put("gram_size", "3");
         ngramProperties.put("bf_size", "256");
         Index indexNgramBf = new Index(4, "ngram_bf_index", Lists.newArrayList("col_4"),
-                        IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4");
+                        IndexType.NGRAM_BF, ngramProperties, "ngram_bf index on col_4", Lists.newArrayList(4));
         indexes.add(indexBitmap);
         indexes.add(indexInverted);
         indexes.add(indexBf);

fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java

@@ -65,7 +65,7 @@ public void testSerialization() throws IOException {
         indexSchemaMap.put(tableId, fullSchema);
 
         List<Index> indexes = Lists.newArrayList(
-                new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx"));
+                new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1)));
 
         TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId,
                 indexSchemaMap, indexes, jobId);

gensrc/thrift/Descriptors.thrift

@@ -227,6 +227,7 @@ struct TOlapTableIndex {
   4: optional string comment
   5: optional i64 index_id
   6: optional map<string, string> properties
+  7: optional list<i32> column_unique_ids
 }
 
 struct TOlapTableIndexSchema {

regression-test/data/inverted_index_p0/index_change/test_index_change_on_renamed_column.out

@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+1	hello world
+2	welcome to the world
+
+-- !select2 --
+1	hello world
+2	welcome to the world
+
+-- !select3 --
+2	welcome to the world

regression-test/suites/inverted_index_p0/test_match_without_index.groovy → regression-test/suites/fault_injection_p0/test_match_without_index_fault_injection.groovy

@@ -16,9 +16,9 @@
 // under the License.
 
 
-suite("test_match_without_index", "p0") {
+suite("test_match_without_index_fault_injection", "nonConcurrent") {
 
-    def testTable = "test_match_without_index"
+    def testTable = "test_match_without_index_fault_injection"
     sql "DROP TABLE IF EXISTS ${testTable}"
     sql """
         CREATE TABLE ${testTable} (