Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix #158: support empty Projection clause; add full decoding of projection definition (but not yet processing) #286

Merged
merged 14 commits into from
Mar 24, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package io.stargate.sgv2.jsonapi.api.model.command;

import com.fasterxml.jackson.databind.JsonNode;
import io.stargate.sgv2.jsonapi.service.projection.DocumentProjector;

/*
* All the commands that need Projection definitions will have to implement this.
*/
public interface Projectable {
JsonNode projectionDefinition();

default DocumentProjector buildProjector() {
return DocumentProjector.createFromDefinition(projectionDefinition());
}
}
Original file line number Diff line number Diff line change
@@ -2,7 +2,9 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.JsonNode;
import io.stargate.sgv2.jsonapi.api.model.command.Filterable;
import io.stargate.sgv2.jsonapi.api.model.command.Projectable;
import io.stargate.sgv2.jsonapi.api.model.command.ReadCommand;
import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.FilterClause;
import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause;
@@ -15,9 +17,10 @@
@JsonTypeName("find")
public record FindCommand(
@Valid @JsonProperty("filter") FilterClause filterClause,
@JsonProperty("projection") JsonNode projectionDefinition,
@Valid @JsonProperty("sort") SortClause sortClause,
@Valid @Nullable Options options)
implements ReadCommand, Filterable {
implements ReadCommand, Filterable, Projectable {

public record Options(
@Valid
Original file line number Diff line number Diff line change
@@ -2,7 +2,9 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.JsonNode;
import io.stargate.sgv2.jsonapi.api.model.command.Filterable;
import io.stargate.sgv2.jsonapi.api.model.command.Projectable;
import io.stargate.sgv2.jsonapi.api.model.command.ReadCommand;
import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.FilterClause;
import io.stargate.sgv2.jsonapi.api.model.command.clause.update.UpdateClause;
@@ -18,9 +20,10 @@
@JsonTypeName("findOneAndUpdate")
public record FindOneAndUpdateCommand(
@Valid @JsonProperty("filter") FilterClause filterClause,
@JsonProperty("projection") JsonNode projectionDefinition,
@NotNull @Valid @JsonProperty("update") UpdateClause updateClause,
@Valid @Nullable Options options)
implements ReadCommand, Filterable {
implements ReadCommand, Filterable, Projectable {

@Schema(
name = "FindOneAndUpdateCommand.Options",
Original file line number Diff line number Diff line change
@@ -2,7 +2,9 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.JsonNode;
import io.stargate.sgv2.jsonapi.api.model.command.Filterable;
import io.stargate.sgv2.jsonapi.api.model.command.Projectable;
import io.stargate.sgv2.jsonapi.api.model.command.ReadCommand;
import io.stargate.sgv2.jsonapi.api.model.command.clause.filter.FilterClause;
import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortClause;
@@ -13,5 +15,6 @@
@JsonTypeName("findOne")
public record FindOneCommand(
@Valid @JsonProperty("filter") FilterClause filterClause,
@JsonProperty("projection") JsonNode projectionDefinition,
@Valid @JsonProperty("sort") SortClause sortClause)
implements ReadCommand, Filterable {}
implements ReadCommand, Filterable, Projectable {}
Original file line number Diff line number Diff line change
@@ -34,6 +34,8 @@ public enum ErrorCode {

UNSUPPORTED_OPERATION("Unsupported operation class"),

UNSUPPORTED_PROJECTION_PARAM("Unsupported projection parameter"),

UNSUPPORTED_UPDATE_DATA_TYPE("Unsupported update data type"),

UNSUPPORTED_UPDATE_OPERATION("Unsupported update operation"),
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.stargate.sgv2.jsonapi.service.operation.model;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.smallrye.mutiny.Uni;
import io.stargate.bridge.grpc.BytesValues;
@@ -10,6 +11,7 @@
import io.stargate.sgv2.jsonapi.exception.JsonApiException;
import io.stargate.sgv2.jsonapi.service.bridge.executor.QueryExecutor;
import io.stargate.sgv2.jsonapi.service.operation.model.impl.ReadDocument;
import io.stargate.sgv2.jsonapi.service.projection.DocumentProjector;
import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId;
import java.util.ArrayList;
import java.util.Iterator;
@@ -18,7 +20,7 @@

/**
* ReadOperation interface which all find command operations will use. It also provides the
* implementation to excute and query and parse the result set as {@link FindResponse}
* implementation to execute and query and parse the result set as {@link FindResponse}
*/
public interface ReadOperation extends Operation {
String[] documentColumns = {"key", "tx_id", "doc_json"};
@@ -41,7 +43,8 @@ default Uni<FindResponse> findDocument(
String pagingState,
int pageSize,
boolean readDocument,
ObjectMapper objectMapper) {
ObjectMapper objectMapper,
DocumentProjector projection) {
return queryExecutor
.executeRead(query, Optional.ofNullable(pagingState), pageSize)
.onItem()
@@ -55,13 +58,16 @@ default Uni<FindResponse> findDocument(
QueryOuterClass.Row row = rowIterator.next();
ReadDocument document = null;
try {
JsonNode root =
readDocument ? objectMapper.readTree(Values.string(row.getValues(2))) : null;
if (root != null) {
projection.applyProjection(root);
}
document =
new ReadDocument(
getDocumentId(row.getValues(0)), // key
Values.uuid(row.getValues(1)), // tx_id
readDocument
? objectMapper.readTree(Values.string(row.getValues(2)))
: null);
root);
} catch (JsonProcessingException e) {
throw new JsonApiException(ErrorCode.DOCUMENT_UNPARSEABLE);
}
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@
import io.stargate.sgv2.jsonapi.service.bridge.executor.QueryExecutor;
import io.stargate.sgv2.jsonapi.service.operation.model.ReadOperation;
import io.stargate.sgv2.jsonapi.service.operation.model.ReadType;
import io.stargate.sgv2.jsonapi.service.projection.DocumentProjector;
import io.stargate.sgv2.jsonapi.service.shredding.model.DocumentId;
import java.util.ArrayList;
import java.util.List;
@@ -24,6 +25,7 @@
public record FindOperation(
CommandContext commandContext,
List<DBFilterBase> filters,
DocumentProjector projection,
String pagingState,
int limit,
int pageSize,
@@ -64,7 +66,8 @@ public Uni<FindResponse> getDocuments(
pagingState,
pageSize,
ReadType.DOCUMENT == readType,
objectMapper);
objectMapper,
projection);
}
default -> {
JsonApiException failure =
Original file line number Diff line number Diff line change
@@ -181,6 +181,11 @@ private Uni<UpdatedDocument> processUpdate(
if (returnDocumentInResponse) {
documentToReturn =
returnUpdatedDocument ? updatedDocument : originalDocument;
// 24-Mar-2023, tatu: Will need to add projection in follow-up PR;
// one we get here is identity-projection which does nothing.
//
// DocumentProjector projector = findOperation().projection();
// projector.applyProjection(documentToReturn);
}
return new UpdatedDocument(
writableShreddedDocument.id(), upsert, documentToReturn, null);
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package io.stargate.sgv2.jsonapi.service.projection;

import com.fasterxml.jackson.databind.JsonNode;
import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
import io.stargate.sgv2.jsonapi.exception.ErrorCode;
import io.stargate.sgv2.jsonapi.exception.JsonApiException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

/**
* Helper class that implements functionality needed to support projections on documents fetched via
* various {@code find} commands.
*/
public class DocumentProjector {
/** Pseudo-projector that makes no modifications to documents */
private static final DocumentProjector IDENTITY_PROJECTOR = new DocumentProjector(null, true);

private final ProjectionLayer rootLayer;

/** Whether this projector is inclusion- ({@code true}) or exclusion ({@code false}) based. */
private final boolean inclusion;

private DocumentProjector(ProjectionLayer rootLayer, boolean inclusion) {
this.rootLayer = rootLayer;
this.inclusion = inclusion;
}

public static DocumentProjector createFromDefinition(JsonNode projectionDefinition) {
if (projectionDefinition == null) {
return identityProjector();
}
if (!projectionDefinition.isObject()) {
throw new JsonApiException(
ErrorCode.UNSUPPORTED_PROJECTION_PARAM,
ErrorCode.UNSUPPORTED_PROJECTION_PARAM.getMessage()
+ ": definition must be OBJECT, was "
+ projectionDefinition.getNodeType());
}
return PathCollector.collectPaths(projectionDefinition).buildProjector();
}

public static DocumentProjector identityProjector() {
return IDENTITY_PROJECTOR;
}

public boolean isInclusion() {
return inclusion;
}

public void applyProjection(JsonNode document) {
if (rootLayer != null) { // null -> identity projection (no-op)
throw new JsonApiException(
ErrorCode.UNSUPPORTED_PROJECTION_PARAM, "Non-identity Projections not yet supported");
}
}

// Mostly for deserialization tests
@Override
public boolean equals(Object o) {
if (o instanceof DocumentProjector) {
DocumentProjector other = (DocumentProjector) o;
return (this.inclusion == other.inclusion) && Objects.equals(this.rootLayer, other.rootLayer);
}
return false;
}

@Override
public int hashCode() {
return rootLayer.hashCode();
}

/**
* Helper object used to traverse and collection inclusion/exclusion path definitions and verify
* that there are only one or the other (except for doc id). Does not build data structures for
* actual matching.
*/
private static class PathCollector {
private List<String> paths = new ArrayList<>();

private int exclusions, inclusions;

private Boolean idInclusion = null;

private PathCollector() {}

static PathCollector collectPaths(JsonNode def) {
return new PathCollector().collectFromObject(def, null);
}

public DocumentProjector buildProjector() {
if (isIdentityProjection()) {
return DocumentProjector.identityProjector();
}

// One more thing: do we need to add document id?
if (inclusions > 0) { // inclusion-based projection
// doc-id included unless explicitly excluded
return new DocumentProjector(
ProjectionLayer.buildLayers(paths, !Boolean.FALSE.equals(idInclusion)), true);
} else { // exclusion-based
// doc-id excluded only if explicitly excluded
return new DocumentProjector(
ProjectionLayer.buildLayers(paths, Boolean.FALSE.equals(idInclusion)), false);
}
}

/**
* Accessor to use for checking if collected paths indicate "empty" (no-operation) projection:
* if so, caller can avoid actual construction or evaluation.
*/
boolean isIdentityProjection() {
// Only the case if we have no non-doc-id inclusions/exclusions AND
// doc-id is included (by default or explicitly)
return paths.isEmpty() && !Boolean.FALSE.equals(idInclusion);
}

PathCollector collectFromObject(JsonNode ob, String parentPath) {
var it = ob.fields();
while (it.hasNext()) {
var entry = it.next();
String path = entry.getKey();
if (parentPath != null) {
path = parentPath + "." + path;
}
JsonNode value = entry.getValue();
if (value.isNumber()) {
// "0" means exclude (like false); any other number include
if (BigDecimal.ZERO.equals(value.decimalValue())) {
addExclusion(path);
} else {
addInclusion(path);
}
} else if (value.isBoolean()) {
if (value.booleanValue()) {
addInclusion(path);
} else {
addExclusion(path);
}
} else if (value.isObject()) {
// Nested definitions allowed, too
collectFromObject(value, path);
} else {
// Unknown JSON node type; error
throw new JsonApiException(
ErrorCode.UNSUPPORTED_PROJECTION_PARAM,
ErrorCode.UNSUPPORTED_PROJECTION_PARAM.getMessage()
+ ": path ('"
+ path
+ "') value must be NUMBER, BOOLEAN or OBJECT, was "
+ value.getNodeType());
}
}
return this;
}

private void addExclusion(String path) {
if (DocumentConstants.Fields.DOC_ID.equals(path)) {
idInclusion = false;
} else {
// Must not mix exclusions and inclusions
if (inclusions > 0) {
throw new JsonApiException(
ErrorCode.UNSUPPORTED_PROJECTION_PARAM,
ErrorCode.UNSUPPORTED_PROJECTION_PARAM.getMessage()
+ ": cannot exclude '"
+ path
+ "' on inclusion projection");
}
++exclusions;
paths.add(path);
}
}

private void addInclusion(String path) {
if (DocumentConstants.Fields.DOC_ID.equals(path)) {
idInclusion = true;
} else {
// Must not mix exclusions and inclusions
if (exclusions > 0) {
throw new JsonApiException(
ErrorCode.UNSUPPORTED_PROJECTION_PARAM,
ErrorCode.UNSUPPORTED_PROJECTION_PARAM.getMessage()
+ ": cannot include '"
+ path
+ "' on exclusion projection");
}
++inclusions;
paths.add(path);
}
}
}
}
Loading