Now unnest allows bound, in and selector filters on the unnested column

docs/tutorials/tutorial-jupyter-index.md

@@ -42,6 +42,7 @@ Make sure you meet the following requirements before starting the Jupyter-based
    pip3 install requests
    ```
 
+
 - JupyterLab (recommended) or Jupyter Notebook running on a non-default port. By default, Druid
   and Jupyter both try to use port `8888`, so start Jupyter on a different port.
 
@@ -99,10 +100,12 @@ The notebooks are located in the [apache/druid repo](https://github.com/apache/d
 
 The links that follow are the raw GitHub URLs, so you can use them to download the notebook directly, such as with `wget`, or manually through your web browser. Note that if you save the file from your web browser, make sure to remove the `.txt` extension.
 
+
 - [Introduction to the Druid REST API](
   https://raw.githubusercontent.com/apache/druid/master/examples/quickstart/jupyter-notebooks/api-tutorial.ipynb)
   walks you through some of the basics related to the Druid REST API and several endpoints.
 - [Introduction to the Druid Python API](
   https://raw.githubusercontent.com/apache/druid/master/examples/quickstart/jupyter-notebooks/Python_API_Tutorial.ipynb)
   walks you through some of the basics related to the Druid API using the Python wrapper API.
+
 - [Introduction to Druid SQL](https://raw.githubusercontent.com/apache/druid/master/examples/quickstart/jupyter-notebooks/sql-tutorial.ipynb) covers the basics of Druid SQL.

processing/src/main/java/org/apache/druid/query/UnnestDataSource.java

@@ -23,13 +23,13 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.planning.DataSourceAnalysis;
 import org.apache.druid.segment.SegmentReference;
 import org.apache.druid.segment.UnnestSegmentReference;
 import org.apache.druid.utils.JvmUtils;
 
 import javax.annotation.Nullable;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -38,7 +38,7 @@
 
 /**
  * The data source for representing an unnest operation.
- *
+ * <p>
  * An unnest data source has the following:
  * a base data source which is to be unnested
  * the column name of the MVD which will be unnested
@@ -50,30 +50,30 @@ public class UnnestDataSource implements DataSource
   private final DataSource base;
   private final String column;
   private final String outputName;
-  private final LinkedHashSet<String> allowList;
+  private final DimFilter outputColumnFilter;
 
   private UnnestDataSource(
       DataSource dataSource,
       String columnName,
       String outputName,
-      LinkedHashSet<String> allowList
+      DimFilter outputColumnFilter
   )
   {
     this.base = dataSource;
     this.column = columnName;
     this.outputName = outputName;
-    this.allowList = allowList;
+    this.outputColumnFilter = outputColumnFilter;
   }
 
   @JsonCreator
   public static UnnestDataSource create(
       @JsonProperty("base") DataSource base,
       @JsonProperty("column") String columnName,
       @JsonProperty("outputName") String outputName,
-      @Nullable @JsonProperty("allowList") LinkedHashSet<String> allowList
+      @Nullable @JsonProperty("outputColumnFilter") DimFilter outputColumnFilter
   )
   {
-    return new UnnestDataSource(base, columnName, outputName, allowList);
+    return new UnnestDataSource(base, columnName, outputName, outputColumnFilter);
   }
 
   @JsonProperty("base")
@@ -94,12 +94,11 @@ public String getOutputName()
     return outputName;
   }
 
-  @JsonProperty("allowList")
-  public LinkedHashSet<String> getAllowList()
+  @JsonProperty("outputColumnFilter")
+  public DimFilter getOutputColumnFilter()
   {
-    return allowList;
+    return outputColumnFilter;
   }
-
   @Override
   public Set<String> getTableNames()
   {
@@ -118,7 +117,7 @@ public DataSource withChildren(List<DataSource> children)
     if (children.size() != 1) {
       throw new IAE("Expected [1] child, got [%d]", children.size());
     }
-    return new UnnestDataSource(children.get(0), column, outputName, allowList);
+    return new UnnestDataSource(children.get(0), column, outputName, outputColumnFilter);
   }
 
   @Override
@@ -163,7 +162,7 @@ public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
                         segmentMapFn.apply(baseSegment),
                         column,
                         outputName,
-                        allowList
+                        outputColumnFilter
                     );
           }
         }
@@ -174,7 +173,7 @@ public Function<SegmentReference, SegmentReference> createSegmentMapFunction(
   @Override
   public DataSource withUpdatedDataSource(DataSource newSource)
   {
-    return new UnnestDataSource(newSource, column, outputName, allowList);
+    return new UnnestDataSource(newSource, column, outputName, outputColumnFilter);
   }
 
   @Override
@@ -213,7 +212,7 @@ public boolean equals(Object o)
   @Override
   public int hashCode()
   {
-    return Objects.hash(base, column, outputName);
+    return Objects.hash(base, column, outputName, outputColumnFilter);
   }
 
   @Override
@@ -223,7 +222,7 @@ public String toString()
            "base=" + base +
            ", column='" + column + '\'' +
            ", outputName='" + outputName + '\'' +
-           ", allowList=" + allowList +
+           ", outputFilter='" + outputColumnFilter + '\'' +
            '}';
   }
 

processing/src/main/java/org/apache/druid/segment/UnnestColumnValueSelectorCursor.java

@@ -22,15 +22,17 @@
 import org.apache.druid.java.util.common.UOE;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.dimension.DimensionSpec;
+import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.ValueMatcher;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
+import org.apache.druid.segment.filter.BooleanValueMatcher;
 import org.joda.time.DateTime;
 
 import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.LinkedHashSet;
 import java.util.List;
 
 /**
@@ -50,7 +52,7 @@
  * unnestCursor.advance() -> 'e'
  * <p>
  * <p>
- * The allowSet if available helps skip over elements which are not in the allowList by moving the cursor to
+ * The filter if available helps skip over elements which are not in the allowList by moving the cursor to
  * the next available match.
  * <p>
  * The index reference points to the index of each row that the unnest cursor is accessing through currentVal
@@ -65,7 +67,7 @@ public class UnnestColumnValueSelectorCursor implements Cursor
   private final ColumnValueSelector columnValueSelector;
   private final String columnName;
   private final String outputName;
-  private final LinkedHashSet<String> allowSet;
+  private final ValueMatcher valueMatcher;
   private int index;
   private Object currentVal;
   private List<Object> unnestListForCurrentRow;
@@ -76,7 +78,7 @@ public UnnestColumnValueSelectorCursor(
       ColumnSelectorFactory baseColumSelectorFactory,
       String columnName,
       String outputColumnName,
-      LinkedHashSet<String> allowSet
+      @Nullable Filter filter
   )
   {
     this.baseCursor = cursor;
@@ -86,7 +88,11 @@ public UnnestColumnValueSelectorCursor(
     this.index = 0;
     this.outputName = outputColumnName;
     this.needInitialization = true;
-    this.allowSet = allowSet;
+    if (filter != null) {
+      this.valueMatcher = filter.makeMatcher(getColumnSelectorFactory());
+    } else {
+      this.valueMatcher = BooleanValueMatcher.of(true);
+    }
   }
 
   @Override
@@ -191,11 +197,7 @@ public boolean isNull()
           public Object getObject()
           {
             if (!unnestListForCurrentRow.isEmpty()) {
-              if (allowSet == null || allowSet.isEmpty()) {
-                return unnestListForCurrentRow.get(index);
-              } else if (allowSet.contains((String) unnestListForCurrentRow.get(index))) {
-                return unnestListForCurrentRow.get(index);
-              }
+              return unnestListForCurrentRow.get(index);
             }
             return null;
           }
@@ -243,9 +245,17 @@ public void advance()
   @Override
   public void advanceUninterruptibly()
   {
-    do {
+    // the index currently points to an element at position i ($e_i)
+    // while $e_i does not match the filter
+    // keep advancing the pointer to the first valid match
+    // calling advanceAndUpdate increments the index position so needs a call to matches() again for new match status
+    while (true) {
       advanceAndUpdate();
-    } while (matchAndProceed());
+      boolean match = valueMatcher.matches();
+      if (match || baseCursor.isDone()) {
+        return;
+      }
+    }
   }
 
   @Override
@@ -311,12 +321,11 @@ private void initialize()
   {
     this.unnestListForCurrentRow = new ArrayList<>();
     getNextRow(needInitialization);
-    if (allowSet != null) {
-      if (!allowSet.isEmpty()) {
-        if (!allowSet.contains((String) unnestListForCurrentRow.get(index))) {
-          advance();
-        }
-      }
+
+    // If the first value the index is pointing to does not match the filter
+    // advance the index to the first value which will match
+    if (!valueMatcher.matches() && !baseCursor.isDone()) {
+      advance();
     }
     needInitialization = false;
   }
@@ -339,22 +348,4 @@ private void advanceAndUpdate()
       index++;
     }
   }
-
-  /**
-   * This advances the unnest cursor in cases where an allowList is specified
-   * and the current value at the unnest cursor is not in the allowList.
-   * The cursor in such cases is moved till the next match is found.
-   *
-   * @return a boolean to indicate whether to stay or move cursor
-   */
-  private boolean matchAndProceed()
-  {
-    boolean matchStatus;
-    if (allowSet == null || allowSet.isEmpty()) {
-      matchStatus = true;
-    } else {
-      matchStatus = allowSet.contains((String) unnestListForCurrentRow.get(index));
-    }
-    return !baseCursor.isDone() && !matchStatus;
-  }
 }