ESQL: Add option to drop null fields

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/SingletonOrdinalsBuilder.java

@@ -43,10 +43,6 @@ public SingletonOrdinalsBuilder appendOrd(int value) {
         return this;
     }
 
-    int[] ords() {
-        return ords;
-    }
-
     @Override
     public SingletonOrdinalsBuilder beginPositionEntry() {
         throw new UnsupportedOperationException("should only have one value per doc");

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockBuilderTests.java

@@ -46,11 +46,11 @@ public BlockBuilderTests(ElementType elementType) {
     }
 
     public void testAllNulls() {
-        for (int numEntries : List.of(1, randomIntBetween(1, 100))) {
+        for (int numEntries : List.of(1, between(1, 100), between(101, 1000))) {
             testAllNullsImpl(elementType.newBlockBuilder(0, blockFactory), numEntries);
-            testAllNullsImpl(elementType.newBlockBuilder(100, blockFactory), numEntries);
-            testAllNullsImpl(elementType.newBlockBuilder(1000, blockFactory), numEntries);
-            testAllNullsImpl(elementType.newBlockBuilder(randomIntBetween(0, 100), blockFactory), numEntries);
+            testAllNullsImpl(elementType.newBlockBuilder(numEntries, blockFactory), numEntries);
+            testAllNullsImpl(elementType.newBlockBuilder(numEntries * 10, blockFactory), numEntries);
+            testAllNullsImpl(elementType.newBlockBuilder(between(0, numEntries), blockFactory), numEntries);
         }
     }
 
@@ -60,17 +60,14 @@ private void testAllNullsImpl(Block.Builder builder, int numEntries) {
         }
         try (Block block = builder.build()) {
             assertThat(block.getPositionCount(), is(numEntries));
-            assertThat(block.isNull(0), is(true));
-            assertThat(block.isNull(numEntries - 1), is(true));
-            assertThat(block.isNull(randomPosition(numEntries)), is(true));
+            for (int p = 0; p < numEntries; p++) {
+                assertThat(block.isNull(p), is(true));
+            }
+            assertThat(block.areAllValuesNull(), is(true));
         }
         assertThat(blockFactory.breaker().getUsed(), equalTo(0L));
     }
 
-    static int randomPosition(int positionCount) {
-        return positionCount == 1 ? 0 : randomIntBetween(0, positionCount - 1);
-    }
-
     public void testCloseWithoutBuilding() {
         elementType.newBlockBuilder(10, blockFactory).close();
         assertThat(blockFactory.breaker().getUsed(), equalTo(0L));

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java

@@ -40,6 +40,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -53,6 +54,7 @@
 import static org.elasticsearch.xpack.ql.util.StringUtils.parseIP;
 
 public class EsqlQueryResponse extends ActionResponse implements ChunkedToXContent, Releasable {
+    public static final String DROP_NULL_COLUMNS_OPTION = "drop_null_columns";
 
     private final List<ColumnInfo> columns;
     private final List<Page> pages;
@@ -120,32 +122,39 @@ public boolean columnar() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params unused) {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        boolean dropNullColumns = params.paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false);
+        boolean[] keepColumns = new boolean[columns.size()];
+        if (dropNullColumns) {
+            for (int c = 0; c < keepColumns.length; c++) {
+                keepColumns[c] = allColumnsAreNull(c) == false;
+                System.err.println(c + " "  + keepColumns[c]);
+            }
+        } else {
+            Arrays.fill(keepColumns, true);
+        }
         final BytesRef scratch = new BytesRef();
         final Iterator<? extends ToXContent> valuesIt;
         if (pages.isEmpty()) {
             valuesIt = Collections.emptyIterator();
         } else if (columnar) {
-            valuesIt = Iterators.flatMap(
-                Iterators.forRange(
-                    0,
-                    columns().size(),
-                    column -> Iterators.concat(
-                        Iterators.single(((builder, params) -> builder.startArray())),
-                        Iterators.flatMap(pages.iterator(), page -> {
-                            ColumnInfo.PositionToXContent toXContent = columns.get(column)
-                                .positionToXContent(page.getBlock(column), scratch);
-                            return Iterators.forRange(
-                                0,
-                                page.getPositionCount(),
-                                position -> (builder, params) -> toXContent.positionToXContent(builder, params, position)
-                            );
-                        }),
-                        ChunkedToXContentHelper.endArray()
-                    )
-                ),
-                Function.identity()
-            );
+            valuesIt = Iterators.flatMap(Iterators.forRange(0, columns().size(), column -> {
+                if (keepColumns[column] == false) {
+                    return Collections.emptyIterator();
+                }
+                return Iterators.concat(
+                    Iterators.single(((builder, p) -> builder.startArray())),
+                    Iterators.flatMap(pages.iterator(), page -> {
+                        ColumnInfo.PositionToXContent toXContent = columns.get(column).positionToXContent(page.getBlock(column), scratch);
+                        return Iterators.forRange(
+                            0,
+                            page.getPositionCount(),
+                            position -> (builder, p) -> toXContent.positionToXContent(builder, p, position)
+                        );
+                    }),
+                    ChunkedToXContentHelper.endArray()
+                );
+            }), Function.identity());
         } else {
             valuesIt = Iterators.flatMap(pages.iterator(), page -> {
                 final int columnCount = columns.size();
@@ -154,24 +163,38 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params unused
                 for (int column = 0; column < columnCount; column++) {
                     toXContents[column] = columns.get(column).positionToXContent(page.getBlock(column), scratch);
                 }
-                return Iterators.forRange(0, page.getPositionCount(), position -> (builder, params) -> {
+                return Iterators.forRange(0, page.getPositionCount(), position -> (builder, p) -> {
                     builder.startArray();
                     for (int c = 0; c < columnCount; c++) {
-                        toXContents[c].positionToXContent(builder, params, position);
+                        if (keepColumns[c]) {
+                            toXContents[c].positionToXContent(builder, p, position);
+                        }
                     }
                     return builder.endArray();
                 });
             });
         }
-        return Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.singleChunk((builder, params) -> {
+        return Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.singleChunk((builder, p) -> {
             builder.startArray("columns");
-            for (ColumnInfo col : columns) {
-                col.toXContent(builder, params);
+            for (int c = 0; c < columns.size(); c++) {
+                if (keepColumns[c]) {
+                    ColumnInfo col = columns.get(c);
+                    col.toXContent(builder, p);
+                }
             }
             return builder.endArray();
         }), ChunkedToXContentHelper.array("values", valuesIt), ChunkedToXContentHelper.endObject());
     }
 
+    private boolean allColumnsAreNull(int c) {
+        for (Page page : pages) {
+            if (page.getBlock(c).areAllValuesNull() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     @Override
     public boolean isFragment() {
         return false;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java

@@ -62,6 +62,6 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
 
     @Override
     protected Set<String> responseParams() {
-        return Collections.singleton(URL_PARAM_DELIMITER);
+        return Set.of(URL_PARAM_DELIMITER, EsqlQueryResponse.DROP_NULL_COLUMNS_OPTION);
     }
 }