-
Notifications
You must be signed in to change notification settings - Fork 138
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add value set skipping index (#1708)
* Add value list skipping index Signed-off-by: Chen Dai <daichen@amazon.com> * Add skipping kind enum and IT Signed-off-by: Chen Dai <daichen@amazon.com> * Add more comments for review Signed-off-by: Chen Dai <daichen@amazon.com> * Add user doc Signed-off-by: Chen Dai <daichen@amazon.com> * Update doc with more details on skipping index Signed-off-by: Chen Dai <daichen@amazon.com> * Update doc to fix markdown format Signed-off-by: Chen Dai <daichen@amazon.com> * Remove unused imports Signed-off-by: Chen Dai <daichen@amazon.com> * Addressed PR comments Signed-off-by: Chen Dai <daichen@amazon.com> --------- Signed-off-by: Chen Dai <daichen@amazon.com>
- Loading branch information
Showing
8 changed files
with
224 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
39 changes: 39 additions & 0 deletions
39
...rc/main/scala/org/opensearch/flint/spark/skipping/valueset/ValueSetSkippingStrategy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.flint.spark.skipping.valueset | ||
|
||
import org.opensearch.flint.spark.skipping.FlintSparkSkippingStrategy | ||
import org.opensearch.flint.spark.skipping.FlintSparkSkippingStrategy.SkippingKind.{SkippingKind, ValuesSet} | ||
|
||
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal, Predicate} | ||
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateFunction, CollectSet} | ||
import org.apache.spark.sql.functions.col | ||
|
||
/** | ||
* Skipping strategy based on unique column value set. | ||
*/ | ||
case class ValueSetSkippingStrategy( | ||
override val kind: SkippingKind = ValuesSet, | ||
override val columnName: String, | ||
override val columnType: String) | ||
extends FlintSparkSkippingStrategy { | ||
|
||
override def outputSchema(): Map[String, String] = | ||
Map(columnName -> columnType) | ||
|
||
override def getAggregators: Seq[AggregateFunction] = | ||
Seq(CollectSet(col(columnName).expr)) | ||
|
||
override def rewritePredicate(predicate: Predicate): Option[Predicate] = { | ||
/* | ||
* This is supposed to be rewritten to ARRAY_CONTAINS(columName, value). | ||
* However, due to push down limitation in Spark, we keep the equation. | ||
*/ | ||
predicate.collect { case EqualTo(AttributeReference(`columnName`, _, _, _), value: Literal) => | ||
EqualTo(col(columnName).expr, value) | ||
}.headOption | ||
} | ||
} |
Oops, something went wrong.