From a84fef0f1383c8ecea9b3ea7a0956a2f351ac950 Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 13:55:49 -0500 Subject: [PATCH 01/10] Added InsertOne and Create collection commands --- pom.xml | 19 ++- .../sgv3/docsapi/StargateDocsV3Api.java | 11 ++ .../docsapi/api/model/command/Command.java | 2 + .../api/model/command/CommandStatus.java | 4 +- .../model/command/SchemaChangeCommand.java | 3 + .../command/impl/CreateCollectionCommand.java | 18 +++ .../sgv3/docsapi/api/v3/DatabaseResource.java | 63 +++++++- .../sgv3/docsapi/exception/ErrorCode.java | 4 +- .../bridge/executor/QueryExecutor.java | 74 +++++++++ .../service/bridge/executor/QueryState.java | 39 +++++ .../serializer/CustomValueSerializers.java | 79 ++++++++++ .../service/operation/model/Operation.java | 4 +- .../model/SchemaChangeOperation.java | 4 + .../model/impl/CreateCollectionOperation.java | 142 +++++++++++++++++ .../operation/model/impl/InsertOperation.java | 61 +++++++- .../model/impl/ModifyOperationPage.java | 34 +++++ .../model/impl/SchemaChangeResult.java | 23 +++ .../service/processor/CommandProcessor.java | 12 +- .../model/impl/CreateCollectionResolver.java | 21 +++ .../docsapi/service/shredding/Shredder.java | 143 +++++++++++++++++- .../service/shredding/model/JsonPath.java | 75 +++++++++ .../service/shredding/model/JsonType.java | 75 +++++++++ .../model/WritableShreddedDocument.java | 25 ++- .../v3/CollectionResourceIntegrationTest.java | 58 ++++++- .../v3/DatabaseResourceIntegrationTest.java | 5 +- 25 files changed, 969 insertions(+), 29 deletions(-) create mode 100644 src/main/java/io/stargate/sgv3/docsapi/api/model/command/SchemaChangeCommand.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/api/model/command/impl/CreateCollectionCommand.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryExecutor.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryState.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/bridge/serializer/CustomValueSerializers.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/operation/model/SchemaChangeOperation.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/CreateCollectionOperation.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/ModifyOperationPage.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/SchemaChangeResult.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/resolver/model/impl/CreateCollectionResolver.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonPath.java create mode 100644 src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonType.java diff --git a/pom.xml b/pom.xml index 27b04cb7d6..6d8bd171ee 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,22 @@ assertj-core test + + org.immutables + value + provided + 2.9.0 + + + com.datastax.oss + java-driver-core + test + + + org.javatuples + javatuples + 1.2 + io.stargate sgv2-quarkus-common @@ -129,12 +145,13 @@ - cassandra-40 + dse-68 true v${stargate.version} + true diff --git a/src/main/java/io/stargate/sgv3/docsapi/StargateDocsV3Api.java b/src/main/java/io/stargate/sgv3/docsapi/StargateDocsV3Api.java index 6ee6f48705..83461acd23 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/StargateDocsV3Api.java +++ b/src/main/java/io/stargate/sgv3/docsapi/StargateDocsV3Api.java @@ -134,6 +134,17 @@ ] } """), + @ExampleObject( + name = "resultCreateCollection", + summary = "Create result", + value = + """ + { + "status": { + "ok": 1 + } + } + """), })) public class StargateDocsV3Api extends Application { diff --git a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/Command.java b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/Command.java index 33cdbef159..ff950e1e90 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/Command.java +++ b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/Command.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.stargate.sgv3.docsapi.api.model.command.impl.CreateCollectionCommand; import io.stargate.sgv3.docsapi.api.model.command.impl.FindOneCommand; import io.stargate.sgv3.docsapi.api.model.command.impl.InsertOneCommand; @@ -27,6 +28,7 @@ include = JsonTypeInfo.As.WRAPPER_OBJECT, property = "commandName") @JsonSubTypes({ + @JsonSubTypes.Type(value = CreateCollectionCommand.class), @JsonSubTypes.Type(value = FindOneCommand.class), @JsonSubTypes.Type(value = InsertOneCommand.class), }) diff --git a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/CommandStatus.java b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/CommandStatus.java index 15b52b4a83..551225c0e7 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/CommandStatus.java +++ b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/CommandStatus.java @@ -4,5 +4,7 @@ public enum CommandStatus { @JsonProperty("insertedIds") - INSERTED_IDS; + INSERTED_IDS, + @JsonProperty("ok") + CREATE_COLLECTION; } diff --git a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/SchemaChangeCommand.java b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/SchemaChangeCommand.java new file mode 100644 index 0000000000..8bb4bc916e --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/SchemaChangeCommand.java @@ -0,0 +1,3 @@ +package io.stargate.sgv3.docsapi.api.model.command; + +public interface SchemaChangeCommand extends Command {} diff --git a/src/main/java/io/stargate/sgv3/docsapi/api/model/command/impl/CreateCollectionCommand.java b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/impl/CreateCollectionCommand.java new file mode 100644 index 0000000000..b1c04ba42b --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/api/model/command/impl/CreateCollectionCommand.java @@ -0,0 +1,18 @@ +package io.stargate.sgv3.docsapi.api.model.command.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.stargate.sgv3.docsapi.api.model.command.SchemaChangeCommand; +import javax.validation.constraints.NotNull; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.media.Schema; + +@Schema(description = "Command that creates a collection.") +@JsonTypeName("createCollection") +public record CreateCollectionCommand( + @NotNull + @Schema( + description = "Name of the collection", + implementation = Object.class, + type = SchemaType.OBJECT) + String name) + implements SchemaChangeCommand {} diff --git a/src/main/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResource.java b/src/main/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResource.java index e108a22631..8e514844ba 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResource.java +++ b/src/main/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResource.java @@ -1,12 +1,30 @@ package io.stargate.sgv3.docsapi.api.v3; import io.smallrye.mutiny.Uni; +import io.stargate.sgv3.docsapi.api.model.command.Command; +import io.stargate.sgv3.docsapi.api.model.command.CommandContext; +import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.api.model.command.impl.CreateCollectionCommand; import io.stargate.sgv3.docsapi.config.constants.OpenApiConstants; +import io.stargate.sgv3.docsapi.service.processor.CommandProcessor; +import javax.inject.Inject; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.ExampleObject; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.parameters.Parameter; +import org.eclipse.microprofile.openapi.annotations.parameters.Parameters; +import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; import org.eclipse.microprofile.openapi.annotations.security.SecurityRequirement; import org.eclipse.microprofile.openapi.annotations.tags.Tag; import org.jboss.resteasy.reactive.RestResponse; @@ -20,8 +38,49 @@ public class DatabaseResource { public static final String BASE_PATH = "/v3/{database}"; + private final CommandProcessor commandProcessor; + + @Inject + public DatabaseResource(CommandProcessor commandProcessor) { + this.commandProcessor = commandProcessor; + } + + @Operation( + summary = "Execute command", + description = "Executes a single command against a collection.") + @Parameters(value = {@Parameter(name = "database", ref = "database")}) + @RequestBody( + content = + @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(anyOf = {CreateCollectionCommand.class}), + examples = { + @ExampleObject(ref = "resultCreateCollection"), + })) + @APIResponses( + @APIResponse( + responseCode = "200", + description = + "Call successful. Returns result of the command execution. Note that in case of errors, response code remains `HTTP 200`.", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = CommandResult.class), + examples = { + @ExampleObject(ref = "resultCreateCollection"), + @ExampleObject(ref = "resultError"), + }))) @POST - public Uni> postCommand() { - return Uni.createFrom().item(RestResponse.ok()); + public Uni> postCommand( + @NotNull @Valid Command command, @PathParam("database") String database) { + + // create context + CommandContext commandContext = new CommandContext(database, null); + + // call processor + return commandProcessor + .processCommand(commandContext, command) + // map to 2xx always + .map(RestResponse::ok); } } diff --git a/src/main/java/io/stargate/sgv3/docsapi/exception/ErrorCode.java b/src/main/java/io/stargate/sgv3/docsapi/exception/ErrorCode.java index dd41a2a711..167ceab984 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/exception/ErrorCode.java +++ b/src/main/java/io/stargate/sgv3/docsapi/exception/ErrorCode.java @@ -4,7 +4,9 @@ public enum ErrorCode { /** Command error codes. */ - COMMAND_NOT_IMPLEMENTED("The provided command is not implemented."); + COMMAND_NOT_IMPLEMENTED("The provided command is not implemented."), + + CREATE_COLLECTION_FAILED("Create collection failed."); private final String message; diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryExecutor.java b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryExecutor.java new file mode 100644 index 0000000000..481bb9592e --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryExecutor.java @@ -0,0 +1,74 @@ +package io.stargate.sgv3.docsapi.service.bridge.executor; + +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.Int32Value; +import io.smallrye.mutiny.Uni; +import io.stargate.bridge.proto.QueryOuterClass; +import io.stargate.sgv2.api.common.StargateRequestInfo; +import io.stargate.sgv2.api.common.config.QueriesConfig; +import java.util.Base64; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; + +@ApplicationScoped +public class QueryExecutor { + + private final QueriesConfig queriesConfig; + + private final StargateRequestInfo requestInfo; + + @Inject + public QueryExecutor(QueriesConfig queriesConfig, StargateRequestInfo requestInfo) { + this.queriesConfig = queriesConfig; + this.requestInfo = requestInfo; + } + + public Uni writeDocument(QueryOuterClass.Query query) { + return queryBridge(query, null); + } + + public Uni execute(QueryOuterClass.Query query) { + return queryBridge(query, null); + } + + private Uni queryBridge( + QueryOuterClass.Query query, String pagingState) { + // construct initial state for the query + BytesValue pagingStateValue = + pagingState != null ? BytesValue.of(ByteString.copyFrom(decodeBase64(pagingState))) : null; + + QueryState state = ImmutableQueryState.of(100, pagingStateValue); + QueryOuterClass.Consistency consistency = queriesConfig.consistency().reads(); + QueryOuterClass.ConsistencyValue.Builder consistencyValue = + QueryOuterClass.ConsistencyValue.newBuilder().setValue(consistency); + QueryOuterClass.QueryParameters.Builder params = + QueryOuterClass.QueryParameters.newBuilder() + .setConsistency(consistencyValue) + .setPageSize(Int32Value.of(state.pageSize())); + + // if we have paging state, set + if (null != state.pagingState()) { + params.setPagingState(state.pagingState()); + } + + // final query is same as the original, just with different params + QueryOuterClass.Query finalQuery = + QueryOuterClass.Query.newBuilder(query).setParameters(params).buildPartial(); + + // execute + return requestInfo + .getStargateBridge() + .executeQuery(finalQuery) + .map( + response -> { + // update next state + QueryOuterClass.ResultSet resultSet = response.getResultSet(); + return resultSet; + }); + } + + protected static byte[] decodeBase64(String base64encoded) { + return Base64.getDecoder().decode(base64encoded); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryState.java b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryState.java new file mode 100644 index 0000000000..36d401adc7 --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/executor/QueryState.java @@ -0,0 +1,39 @@ +package io.stargate.sgv3.docsapi.service.bridge.executor; + +import com.google.protobuf.BytesValue; +import javax.annotation.Nullable; +import org.immutables.value.Value; + +/** + * Utility used by the {@link QueryExecutor} in order to track the state for executing the same + * query in the paginated mode. + */ +@Value.Immutable +public interface QueryState { + + /** @return Current page size to be used with the query. */ + @Value.Parameter + int pageSize(); + + /** + * @return Current paging state to be used with the query. Can be null to denote no + * paging state exists. + */ + @Value.Parameter + @Nullable + BytesValue pagingState(); + + /** + * Constructs the next state for the query. + * + * @param nextPagingState Updated paging state, usually received from the result set. + * @param exponentPageSize If current page size should be increased exponentially. + * @param maxPageSize The absolute max page size that should never be exceeded. + * @return Next {@link QueryState}. + */ + default QueryState next(BytesValue nextPagingState, boolean exponentPageSize, int maxPageSize) { + int nextPageSize = exponentPageSize ? Math.min(pageSize() * 2, maxPageSize) : pageSize(); + + return ImmutableQueryState.of(nextPageSize, nextPagingState); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/bridge/serializer/CustomValueSerializers.java b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/serializer/CustomValueSerializers.java new file mode 100644 index 0000000000..67d109aff3 --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/bridge/serializer/CustomValueSerializers.java @@ -0,0 +1,79 @@ +package io.stargate.sgv3.docsapi.service.bridge.serializer; + +import io.stargate.bridge.grpc.Values; +import io.stargate.bridge.proto.QueryOuterClass; +import io.stargate.sgv3.docsapi.service.shredding.model.JsonPath; +import io.stargate.sgv3.docsapi.service.shredding.model.JsonType; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.javatuples.Pair; + +public class CustomValueSerializers { + public static Map getRawDataValue( + Map> from) { + final Map to = new HashMap<>(from.size()); + for (Map.Entry> entry : from.entrySet()) { + QueryOuterClass.Value key = Values.of(entry.getKey().getPath()); + QueryOuterClass.Value valueTuple = getTupleValue(entry.getValue()); + to.put(key, valueTuple); + } + return to; + } + + private static QueryOuterClass.Value getTupleValue(Pair value) { + List decoded = new ArrayList<>(); + decoded.add(Values.of(value.getValue0().value)); + decoded.add(Values.of(value.getValue1())); + return Values.of(decoded); + } + + public static Map getIntegerMapValues( + Map from) { + final Map to = new HashMap<>(from.size()); + for (Map.Entry entry : from.entrySet()) { + to.put(Values.of(entry.getKey().getPath()), Values.of(entry.getValue())); + } + return to; + } + + public static Set getSetValue(Set from) { + return from.stream().map(val -> Values.of(val.getPath())).collect(Collectors.toSet()); + } + + public static List getListValue(List from) { + return from.stream().map(val -> Values.of(val.getPath())).collect(Collectors.toList()); + } + + public static Map getStringMapValues( + Map from) { + final Map to = new HashMap<>(from.size()); + for (Map.Entry entry : from.entrySet()) { + to.put(Values.of(entry.getKey().getPath()), Values.of(entry.getValue())); + } + return to; + } + + public static Map getBooleanMapValues( + Map from) { + final Map to = new HashMap<>(from.size()); + for (Map.Entry entry : from.entrySet()) { + to.put(Values.of(entry.getKey().getPath()), Values.of(entry.getValue())); + } + return to; + } + + public static Map getDoubleMapValues( + Map from) { + final Map to = new HashMap<>(from.size()); + for (Map.Entry entry : from.entrySet()) { + to.put(Values.of(entry.getKey().getPath()), Values.of(entry.getValue())); + } + return to; + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/Operation.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/Operation.java index 2999d26633..324aab553d 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/Operation.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/Operation.java @@ -1,9 +1,9 @@ package io.stargate.sgv3.docsapi.service.operation.model; import io.smallrye.mutiny.Uni; -import io.stargate.bridge.proto.StargateBridge; import io.stargate.sgv3.docsapi.api.model.command.CommandContext; import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.service.bridge.executor.QueryExecutor; import java.util.function.Supplier; /** @@ -29,5 +29,5 @@ public interface Operation { /** @return The context of the command responsible for this operation. */ CommandContext commandContext(); - Uni> execute(StargateBridge bridge); + Uni> execute(QueryExecutor queryExecutor); } diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/SchemaChangeOperation.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/SchemaChangeOperation.java new file mode 100644 index 0000000000..7174e39e85 --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/SchemaChangeOperation.java @@ -0,0 +1,4 @@ +package io.stargate.sgv3.docsapi.service.operation.model; + +/** Interface for operations that does schema change. */ +public interface SchemaChangeOperation {} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/CreateCollectionOperation.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/CreateCollectionOperation.java new file mode 100644 index 0000000000..d64c0edbc8 --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/CreateCollectionOperation.java @@ -0,0 +1,142 @@ +package io.stargate.sgv3.docsapi.service.operation.model.impl; + +import io.smallrye.mutiny.Uni; +import io.stargate.bridge.proto.QueryOuterClass; +import io.stargate.sgv3.docsapi.api.model.command.CommandContext; +import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.exception.DocsException; +import io.stargate.sgv3.docsapi.exception.ErrorCode; +import io.stargate.sgv3.docsapi.service.bridge.executor.QueryExecutor; +import io.stargate.sgv3.docsapi.service.operation.model.Operation; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public record CreateCollectionOperation(CommandContext commandContext, String name) + implements Operation { + + @Override + public Uni> execute(QueryExecutor queryExecutor) { + final Uni execute = + queryExecutor.execute(getCreateTable(commandContext.database(), name)); + final Uni indexResult = + execute + .onItem() + .transformToUni( + res -> { + final List indexStatements = + getIndexStatements(commandContext.database(), name); + List> indexes = new ArrayList<>(10); + indexStatements.stream() + .forEach(index -> indexes.add(queryExecutor.execute(index))); + return Uni.combine().all().unis(indexes).combinedWith(results -> true); + }); + return indexResult + .onFailure() + .transform( + t -> { + return new DocsException(ErrorCode.CREATE_COLLECTION_FAILED, t); + }) + .onItem() + .transform(res -> SchemaChangeResult.from(res)); + } + + protected QueryOuterClass.Query getCreateTable(String keyspace, String table) { + String createTable = + "CREATE TABLE IF NOT EXISTS %s.%s (" + + " key text," + + " tx_id timeuuid, " + + " doc_properties map," + + " exist_keys set," + + " sub_doc_equals set," + + " array_size map," + + " array_equals map," + + " array_contains set," + + " query_bool_values map," + + " query_dbl_values map," + + " query_text_values map, " + + " query_null_values set, " + + " doc_field_order list, " + + " doc_atomic_fields map>," + + " PRIMARY KEY (key))"; + return QueryOuterClass.Query.newBuilder() + .setCql(String.format(createTable, keyspace, table)) + .build(); + } + + protected List getIndexStatements(String keyspace, String table) { + List statements = new ArrayList<>(10); + + String propertyIndex = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_doc_properties ON %s.%s (entries(doc_properties)) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(propertyIndex, table, keyspace, table)) + .build()); + + String existKeys = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_exists_keys ON %s.%s (exist_keys) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(existKeys, table, keyspace, table)) + .build()); + + String subDocEquals = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_sub_doc_equals ON %s.%s (sub_doc_equals) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(subDocEquals, table, keyspace, table)) + .build()); + + String arraySize = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_array_size ON %s.%s (entries(array_size)) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(arraySize, table, keyspace, table)) + .build()); + + String arrayEquals = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_array_equals ON %s.%s (entries(array_equals)) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(arrayEquals, table, keyspace, table)) + .build()); + + String arrayContains = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_array_contains ON %s.%s (array_contains) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(arrayContains, table, keyspace, table)) + .build()); + + String boolQuery = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_query_bool_values ON %s.%s (entries(query_bool_values)) USING 'StorageAttachedIndex'"; + /*statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(boolQuery, table, keyspace, table)) + .build());*/ + + String dblQuery = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_query_dbl_values ON %s.%s (entries(query_dbl_values)) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(dblQuery, table, keyspace, table)) + .build()); + + String textQuery = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_query_text_values ON %s.%s (entries(query_text_values)) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(textQuery, table, keyspace, table)) + .build()); + + String nullQuery = + "CREATE CUSTOM INDEX IF NOT EXISTS %s_query_null_values ON %s.%s (query_null_values) USING 'StorageAttachedIndex'"; + statements.add( + QueryOuterClass.Query.newBuilder() + .setCql(String.format(nullQuery, table, keyspace, table)) + .build()); + + return statements; + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java index 168eb3c353..228b7bd727 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java @@ -1,12 +1,15 @@ package io.stargate.sgv3.docsapi.service.operation.model.impl; +import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; -import io.stargate.bridge.proto.StargateBridge; +import io.stargate.bridge.grpc.Values; +import io.stargate.bridge.proto.QueryOuterClass; import io.stargate.sgv3.docsapi.api.model.command.CommandContext; import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.service.bridge.executor.QueryExecutor; +import io.stargate.sgv3.docsapi.service.bridge.serializer.CustomValueSerializers; import io.stargate.sgv3.docsapi.service.operation.model.Operation; import io.stargate.sgv3.docsapi.service.shredding.model.WritableShreddedDocument; -import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -20,10 +23,56 @@ public InsertOperation(CommandContext commandContext, WritableShreddedDocument d /** {@inheritDoc} */ @Override - public Uni> execute(StargateBridge bridge) { + public Uni> execute(QueryExecutor queryExecutor) { // TODO implement me - Supplier supplier = - () -> new CommandResult(new CommandResult.ResponseData(Collections.emptyList())); - return Uni.createFrom().>item(supplier); + QueryOuterClass.Query query = buildInsertQuery(); + final Uni> ids = + Multi.createBy() + .concatenating() + .streams(Multi.createFrom().items(documents.stream())) + .onItem() + .transformToUniAndConcatenate(doc -> insertDocument(queryExecutor, query, doc)) + .collect() + .asList(); + return ids.onItem().transform(insertedIds -> ModifyOperationPage.from(insertedIds, documents)); + } + + private static Uni insertDocument( + QueryExecutor queryExecutor, QueryOuterClass.Query query, WritableShreddedDocument doc) { + query = bindInsertValues(query, doc); + return queryExecutor.writeDocument(query).onItem().transform(result -> doc.key()); + } + + private QueryOuterClass.Query buildInsertQuery() { + String insert = + "INSERT INTO %s.%s" + + " (key, tx_id, doc_properties, exist_keys, sub_doc_equals, array_size, array_equals, array_contains, query_bool_values, query_dbl_values , query_text_values, query_null_values, doc_field_order, doc_atomic_fields)" + + " VALUES" + + " (?, now(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + return QueryOuterClass.Query.newBuilder() + .setCql(String.format(insert, commandContext.database(), commandContext.collection())) + .build(); + } + + private static QueryOuterClass.Query bindInsertValues( + QueryOuterClass.Query builtQuery, WritableShreddedDocument doc) { + // respect the order in the DocsApiConstants.ALL_COLUMNS_NAMES + QueryOuterClass.Values.Builder values = + QueryOuterClass.Values.newBuilder() + .addValues(Values.of(doc.key())) + .addValues(Values.of(CustomValueSerializers.getIntegerMapValues(doc.properties()))) + .addValues(Values.of(CustomValueSerializers.getSetValue(doc.existKeys()))) + .addValues(Values.of(CustomValueSerializers.getStringMapValues(doc.subDocEquals()))) + .addValues(Values.of(CustomValueSerializers.getIntegerMapValues(doc.arraySize()))) + .addValues(Values.of(CustomValueSerializers.getStringMapValues(doc.arrayEquals()))) + .addValues(Values.of(CustomValueSerializers.getStringMapValues(doc.arrayContains()))) + .addValues(Values.of(CustomValueSerializers.getBooleanMapValues(doc.queryBoolValues()))) + .addValues( + Values.of(CustomValueSerializers.getDoubleMapValues(doc.queryNumberValues()))) + .addValues(Values.of(CustomValueSerializers.getStringMapValues(doc.queryTextValues()))) + .addValues(Values.of(CustomValueSerializers.getSetValue(doc.queryNullValues()))) + .addValues(Values.of(CustomValueSerializers.getListValue(doc.docFieldOrder()))) + .addValues(Values.of(CustomValueSerializers.getRawDataValue(doc.docAtomicFields()))); + return QueryOuterClass.Query.newBuilder(builtQuery).setValues(values).build(); } } diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/ModifyOperationPage.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/ModifyOperationPage.java new file mode 100644 index 0000000000..ce8cac121e --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/ModifyOperationPage.java @@ -0,0 +1,34 @@ +package io.stargate.sgv3.docsapi.service.operation.model.impl; + +import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.api.model.command.CommandStatus; +import io.stargate.sgv3.docsapi.service.shredding.model.WritableShreddedDocument; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * The internal to modification operation results, what were the ID's of the docs we changed and + * what change. + */ +public class ModifyOperationPage implements Supplier { + + public final List insertedIds; + public final List insertedDocs; + + private ModifyOperationPage( + List insertedIds, List insertedDocs) { + this.insertedIds = insertedIds; + this.insertedDocs = insertedDocs; + } + + public static ModifyOperationPage from( + List insertedIds, List insertedDocs) { + return new ModifyOperationPage(insertedIds, insertedDocs); + } + + @Override + public CommandResult get() { + return new CommandResult(Map.of(CommandStatus.INSERTED_IDS, insertedIds)); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/SchemaChangeResult.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/SchemaChangeResult.java new file mode 100644 index 0000000000..480f95d739 --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/SchemaChangeResult.java @@ -0,0 +1,23 @@ +package io.stargate.sgv3.docsapi.service.operation.model.impl; + +import io.stargate.sgv3.docsapi.api.model.command.CommandResult; +import io.stargate.sgv3.docsapi.api.model.command.CommandStatus; +import java.util.Map; +import java.util.function.Supplier; + +public class SchemaChangeResult implements Supplier { + public final boolean schemaChanged; + + private SchemaChangeResult(boolean schemaChanged) { + this.schemaChanged = schemaChanged; + } + + public static SchemaChangeResult from(boolean schemaChanged) { + return new SchemaChangeResult(schemaChanged); + } + + @Override + public CommandResult get() { + return new CommandResult(Map.of(CommandStatus.CREATE_COLLECTION, schemaChanged ? 1 : 0)); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/processor/CommandProcessor.java b/src/main/java/io/stargate/sgv3/docsapi/service/processor/CommandProcessor.java index 7332bf784f..843dd8ca5e 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/processor/CommandProcessor.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/processor/CommandProcessor.java @@ -1,13 +1,12 @@ package io.stargate.sgv3.docsapi.service.processor; -import io.quarkus.grpc.GrpcClient; import io.smallrye.mutiny.Uni; -import io.stargate.bridge.proto.StargateBridge; import io.stargate.sgv3.docsapi.api.model.command.Command; import io.stargate.sgv3.docsapi.api.model.command.CommandContext; import io.stargate.sgv3.docsapi.api.model.command.CommandResult; import io.stargate.sgv3.docsapi.exception.DocsException; import io.stargate.sgv3.docsapi.exception.mappers.ThrowableCommandResultSupplier; +import io.stargate.sgv3.docsapi.service.bridge.executor.QueryExecutor; import io.stargate.sgv3.docsapi.service.operation.model.Operation; import io.stargate.sgv3.docsapi.service.resolver.CommandResolverService; import java.util.function.Supplier; @@ -26,15 +25,14 @@ @ApplicationScoped public class CommandProcessor { - private final StargateBridge stargateBridge; + private final QueryExecutor queryExecutor; private final CommandResolverService commandResolverService; @Inject public CommandProcessor( - @GrpcClient("bridge") StargateBridge stargateBridge, - CommandResolverService commandResolverService) { - this.stargateBridge = stargateBridge; + QueryExecutor queryExecutor, CommandResolverService commandResolverService) { + this.queryExecutor = queryExecutor; this.commandResolverService = commandResolverService; } @@ -57,7 +55,7 @@ public Uni processCommand( resolver -> { // if we have resolver, resolve operation and execute Operation operation = resolver.resolveCommand(commandContext, command); - return operation.execute(stargateBridge); + return operation.execute(queryExecutor); }) // handler failures here diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/resolver/model/impl/CreateCollectionResolver.java b/src/main/java/io/stargate/sgv3/docsapi/service/resolver/model/impl/CreateCollectionResolver.java new file mode 100644 index 0000000000..896da579df --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/resolver/model/impl/CreateCollectionResolver.java @@ -0,0 +1,21 @@ +package io.stargate.sgv3.docsapi.service.resolver.model.impl; + +import io.stargate.sgv3.docsapi.api.model.command.CommandContext; +import io.stargate.sgv3.docsapi.api.model.command.impl.CreateCollectionCommand; +import io.stargate.sgv3.docsapi.service.operation.model.Operation; +import io.stargate.sgv3.docsapi.service.operation.model.impl.CreateCollectionOperation; +import io.stargate.sgv3.docsapi.service.resolver.model.CommandResolver; +import javax.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class CreateCollectionResolver implements CommandResolver { + @Override + public Class getCommandClass() { + return CreateCollectionCommand.class; + } + + @Override + public Operation resolveCommand(CommandContext ctx, CreateCollectionCommand command) { + return new CreateCollectionOperation(ctx, command.name()); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/Shredder.java b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/Shredder.java index 43350b029e..65e6227cfb 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/Shredder.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/Shredder.java @@ -1,9 +1,28 @@ package io.stargate.sgv3.docsapi.service.shredding; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import io.stargate.sgv3.docsapi.service.shredding.model.JsonPath; +import io.stargate.sgv3.docsapi.service.shredding.model.JsonType; import io.stargate.sgv3.docsapi.service.shredding.model.WritableShreddedDocument; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.Spliterators; +import java.util.UUID; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.enterprise.context.ApplicationScoped; import javax.validation.constraints.NotNull; +import org.javatuples.Pair; /** * Shred an incoming JSON document into the data we need to store in the DB, and then de-shred. @@ -14,7 +33,7 @@ */ @ApplicationScoped public class Shredder { - + private static final Charset CHARSET = Charset.forName("UTF-8"); /** * Shreds a single JSON node into a {@link WritableShreddedDocument} representation. * @@ -22,7 +41,125 @@ public class Shredder { * @return WritableShreddedDocument */ public WritableShreddedDocument shred(@NotNull JsonNode document) { - // TODO @tatu implement me - return new WritableShreddedDocument(); + + // This is where we do all the shredding based on the python code + + // TODO HACK HACK HACK- this is not the correct shredding, only handles top level keys of + // string, number, bool, and null + // also the code is kind of shit - signed, the author + + JsonNode idNode = document.get("_id"); + Optional txId = Optional.empty(); + if (idNode == null || idNode.getNodeType() != JsonNodeType.STRING) { + // TODO Need to confirm the _id type and handling + throw new RuntimeException("Bad _id type or missing"); + } + String id = idNode.asText(); + + // couple of helper functions to build the streams + // whe we get smarter can do more in a single stream, for now still working out what we need to + // do for each shredded field + // its inefficient for now, but am focused on the features + Function, Stream>> streamNodes = + (Set nodeTypes) -> { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(document.fields(), 0), false) + .filter(entry -> nodeTypes.contains(entry.getValue().getNodeType())); + }; + + Supplier> streamFieldNames = + () -> { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(document.fieldNames(), 0), false); + }; + + // Building the fields of the shredded doc + + Map properties = Map.of(); + + // HACK only expects to find top level keys, no sub docs or arrays + Set existKeys = + streamFieldNames.get().map(JsonPath::from).collect(Collectors.toSet()); + + Map subDocEquals = Map.of(); // TODO + Map arraySize = Map.of(); // TODO + Map arrayEquals = Map.of(); // TODO + Map arrayContains = Map.of(); // TODO + + Map queryBoolValues = + streamNodes + .apply(EnumSet.of(JsonNodeType.BOOLEAN)) + .collect( + Collectors.toMap( + entry -> JsonPath.from(entry.getKey()), entry -> entry.getValue().asBoolean())); + + Map queryNumberValues = + streamNodes + .apply(EnumSet.of(JsonNodeType.NUMBER)) + .collect( + Collectors.toMap( + entry -> JsonPath.from(entry.getKey()), + entry -> BigDecimal.valueOf(entry.getValue().asDouble()).stripTrailingZeros())); + + Map queryTextValues = + streamNodes + .apply(EnumSet.of(JsonNodeType.STRING)) + .collect( + Collectors.toMap( + entry -> JsonPath.from(entry.getKey()), entry -> entry.getValue().asText())); + + Set queryNullValues = + streamNodes + .apply(EnumSet.of(JsonNodeType.NULL)) + .map(Map.Entry::getKey) + .map(JsonPath::from) + .collect(Collectors.toSet()); + + List docFieldOrder = + streamFieldNames.get().map(JsonPath::from).collect(Collectors.toList()); + + Map> docAtomicFields = + streamNodes + .apply(EnumSet.of(JsonNodeType.BOOLEAN, JsonNodeType.NUMBER, JsonNodeType.STRING)) + .collect( + Collectors.toMap( + entry -> JsonPath.from(entry.getKey()), + entry -> encodeAtomicValue(entry.getValue()))); + + return new WritableShreddedDocument( + id, + txId, + docFieldOrder, + docAtomicFields, + properties, + existKeys, + subDocEquals, + arraySize, + arrayEquals, + arrayContains, + queryBoolValues, + queryNumberValues, + queryTextValues, + queryNullValues); + } + + public static Pair encodeAtomicValue(JsonNode node) { + + JsonType jsonType = JsonType.fromJacksonType(node.getNodeType()); + + switch (jsonType) { + case STRING: + return Pair.with(jsonType, CHARSET.encode(node.asText())); + case BOOLEAN: + return Pair.with( + jsonType, + node.asBoolean() ? ByteBuffer.wrap(new byte[] {1}) : ByteBuffer.wrap(new byte[] {0})); + case NUMBER: + // TODO HACK - better decimal encoding + BigDecimal decimal = BigDecimal.valueOf(node.asDouble()); + return Pair.with(jsonType, CHARSET.encode(node.asText())); + default: + throw new RuntimeException("Unknown jsonType to encode buffer " + jsonType.toString()); + } } } diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonPath.java b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonPath.java new file mode 100644 index 0000000000..fcceb52f5c --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonPath.java @@ -0,0 +1,75 @@ +package io.stargate.sgv3.docsapi.service.shredding.model; + +import com.google.common.base.Splitter; +import java.util.Objects; + +/** + * Model for any path into the JSON document. Not *true* JsonPath as it is not designed for all + * querying https://github.com/json-path/JsonPath + * + *

Examples are "_id" or "user.fist_name" + * + *

The python lab overloads this with ideas like "my_array.$size" + * + *

NOTE: this is a partial implementation of what was in the python lab, more will be needed to + * do the full shredding work. + */ +public class JsonPath { + + private static final Splitter PATH_SPLITTER = Splitter.on(".").trimResults().omitEmptyStrings(); + public static final JsonPath ROOT = new JsonPath(null, "$"); + + private JsonPath prev; + private String name; + private String path; + + public JsonPath(JsonPath prev, String name) { + this.prev = prev; + this.name = name; + + // Build the path now because it is immutable + // We do not include the root "$" in the path, that is something for queries not what we need in + // the DB + + if (prev != null && !JsonPath.ROOT.equals(prev)) { + path = String.join(".", prev.path, name); + } else { + path = name; + } + } + + public static JsonPath from(String path) { + JsonPath current = null; + + for (String part : PATH_SPLITTER.splitToList(path)) { + current = (current == null) ? new JsonPath(JsonPath.ROOT, part) : new JsonPath(current, part); + } + assert current != null; + return current; + } + + @Override + public int hashCode() { + return Objects.hash(path); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + JsonPath other = (JsonPath) obj; + if (path == null) { + if (other.path != null) return false; + } else if (!path.equals(other.path)) return false; + return true; + } + + public String getName() { + return name; + } + + public String getPath() { + return path; + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonType.java b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonType.java new file mode 100644 index 0000000000..c8d6c5625a --- /dev/null +++ b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/JsonType.java @@ -0,0 +1,75 @@ +package io.stargate.sgv3.docsapi.service.shredding.model; + +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.google.common.base.Functions; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +/** Data type for a JSON value, used for de/encoding and storage. */ +public enum JsonType { + UNKNOWN(-1, JsonNodeType.MISSING), // TODO : Ask Tatu how missing is used + BOOLEAN(1, JsonNodeType.BOOLEAN), + NUMBER(2, JsonNodeType.NUMBER), + STRING(3, JsonNodeType.STRING), + NULL(4, JsonNodeType.NULL), + SUB_DOC(5, JsonNodeType.OBJECT), + ARRAY(6, JsonNodeType.ARRAY); + + public final int value; + public final JsonNodeType jacksonType; + + private JsonType(int value, JsonNodeType jacksonType) { + this.value = value; + this.jacksonType = jacksonType; + } + + private static final Map valueMap; + private static final Map jacksonMap; + + static { + valueMap = + Arrays.stream(values()).collect(Collectors.toMap(e -> e.value, Functions.identity())); + + jacksonMap = + Arrays.stream(values()).collect(Collectors.toMap(e -> e.jacksonType, Functions.identity())); + } + + public static JsonType fromValue(Integer value) { + + JsonType element = valueMap.get(value); + if (element != null) { + return element; + } + throw new RuntimeException("Known Json Type value " + value.toString()); + } + + public static JsonType fromJacksonType(JsonNodeType nodeType) { + + JsonType element = jacksonMap.get(nodeType); + if (element != null) { + return element; + } + throw new RuntimeException("Known Json Type jacksonType " + nodeType.toString()); + } + + public static JsonType typeForValue(Object value) { + // Still need this and the jackson map for handling creating filter clause in code where we dont + // have the JsonNode from jackson + + if (value instanceof String) { + return JsonType.STRING; + } + if (value instanceof Number) { + return JsonType.NUMBER; + } + if (value instanceof Boolean) { + return JsonType.BOOLEAN; + } + if (value == null) { + return JsonType.NULL; + } + + throw new RuntimeException(String.format("Unknown type %s", value)); + } +} diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/WritableShreddedDocument.java b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/WritableShreddedDocument.java index 8399fd748a..3648ce26dc 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/WritableShreddedDocument.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/shredding/model/WritableShreddedDocument.java @@ -1,4 +1,27 @@ package io.stargate.sgv3.docsapi.service.shredding.model; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import org.javatuples.Pair; + /** The fully shredded document, everything we need to write the document. */ -public record WritableShreddedDocument() {} +public record WritableShreddedDocument( + String key, + Optional txID, + List docFieldOrder, + Map> docAtomicFields, + Map properties, + Set existKeys, + Map subDocEquals, + Map arraySize, + Map arrayEquals, + Map arrayContains, + Map queryBoolValues, + Map queryNumberValues, + Map queryTextValues, + Set queryNullValues) {} diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java index acd3da2b39..f7fa403551 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java @@ -11,21 +11,48 @@ import io.restassured.RestAssured; import io.restassured.http.ContentType; import io.stargate.sgv2.api.common.config.constants.HttpConstants; +import io.stargate.sgv2.common.CqlEnabledIntegrationTestBase; import io.stargate.sgv2.common.testresource.StargateTestResource; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.condition.DisabledIfSystemProperty; @QuarkusIntegrationTest @QuarkusTestResource(StargateTestResource.class) -class CollectionResourceIntegrationTest { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CollectionResourceIntegrationTest extends CqlEnabledIntegrationTestBase { + private String collectionName = "col" + RandomStringUtils.randomNumeric(16); @BeforeAll public static void enableLog() { RestAssured.enableLoggingOfRequestAndResponseIfValidationFails(); } + @Test + public final void createCollection() { + String json = + String.format( + """ + { + "createCollection": { + "name": %s + } + } + """, + collectionName); + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(DatabaseResource.BASE_PATH, keyspaceId.asInternal()) + .then() + .statusCode(200); + } + @Nested class FindOne { @@ -45,7 +72,7 @@ public void happyPath() { .contentType(ContentType.JSON) .body(json) .when() - .post(CollectionResource.BASE_PATH, "database", "collection") + .post(CollectionResource.BASE_PATH, keyspaceId.asInternal(), collectionName) .then() .statusCode(200) .body("errors", is(not(empty()))) @@ -56,6 +83,29 @@ public void happyPath() { @Nested class InsertOne { + @Test + public void insertDocument() { + String json = + """ + { + "insertOne": { + "document": { + "_id": "doc1", + "username": "aaron" + } + } + } + """; + + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .body(json) + .when() + .post(CollectionResource.BASE_PATH, keyspaceId.asInternal(), collectionName) + .then() + .statusCode(200); + } @Test public void emptyDocument() { @@ -74,7 +124,7 @@ public void emptyDocument() { .contentType(ContentType.JSON) .body(json) .when() - .post(CollectionResource.BASE_PATH, "database", "collection") + .post(CollectionResource.BASE_PATH, keyspaceId.asInternal(), collectionName) .then() .statusCode(200); } @@ -99,7 +149,7 @@ public void notValidDocumentMissing() { .contentType(ContentType.JSON) .body(json) .when() - .post(CollectionResource.BASE_PATH, "database", "collection") + .post(CollectionResource.BASE_PATH, keyspaceId.asInternal(), collectionName) .then() .statusCode(400); } diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index 66bf4d8990..883b3b0318 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -8,14 +8,17 @@ import io.restassured.RestAssured; import io.restassured.http.ContentType; import io.stargate.sgv2.api.common.config.constants.HttpConstants; +import io.stargate.sgv2.common.CqlEnabledIntegrationTestBase; import io.stargate.sgv2.common.testresource.StargateTestResource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; @QuarkusIntegrationTest @QuarkusTestResource(StargateTestResource.class) -class DatabaseResourceIntegrationTest { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class DatabaseResourceIntegrationTest extends CqlEnabledIntegrationTestBase { @BeforeAll public static void enableLog() { From bd91264efb18ce1ad8661d1231b85a40182aa7f9 Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 14:26:08 -0500 Subject: [PATCH 02/10] Fixed the query error --- .../sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java index f7fa403551..9026148ce8 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/CollectionResourceIntegrationTest.java @@ -38,7 +38,7 @@ public final void createCollection() { """ { "createCollection": { - "name": %s + "name": "%s" } } """, From eda9de3fcb2a514c663e4a20ca8604891232790a Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 15:57:20 -0500 Subject: [PATCH 03/10] Removed the TODO on InsertOperation --- .../docsapi/service/operation/model/impl/InsertOperation.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java index 228b7bd727..87a0b44f7b 100644 --- a/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java +++ b/src/main/java/io/stargate/sgv3/docsapi/service/operation/model/impl/InsertOperation.java @@ -24,7 +24,6 @@ public InsertOperation(CommandContext commandContext, WritableShreddedDocument d /** {@inheritDoc} */ @Override public Uni> execute(QueryExecutor queryExecutor) { - // TODO implement me QueryOuterClass.Query query = buildInsertQuery(); final Uni> ids = Multi.createBy() From e4cfce294bd3f56da805162f0e3a268ea8662170 Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 21:05:10 -0500 Subject: [PATCH 04/10] Make dse profile default --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 25daf03643..52bedda22c 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,13 @@ true + datastax/dse-server + 6.8.28 + stargateio/coordinator-dse-68 v${stargate.version} + dse-${stargate.int-test.cassandra.image-tag}-cluster + 6.8 + true true From 6d9bcd8942ff87d7d2afc709bd8b985ab4c9c69e Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 21:17:06 -0500 Subject: [PATCH 05/10] Fixed invalid integration test --- .../v3/DatabaseResourceIntegrationTest.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index 883b3b0318..41c53d6898 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -2,6 +2,7 @@ import static io.restassured.RestAssured.given; import static io.stargate.sgv2.common.IntegrationTestUtils.getAuthToken; +import static org.hamcrest.Matchers.is; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; @@ -10,6 +11,7 @@ import io.stargate.sgv2.api.common.config.constants.HttpConstants; import io.stargate.sgv2.common.CqlEnabledIntegrationTestBase; import io.stargate.sgv2.common.testresource.StargateTestResource; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -30,13 +32,36 @@ class PostCommand { @Test public void happyPath() { + String json = + String.format( + """ + { + "createCollection": { + "name": "%s" + } + } + """, + "col" + RandomStringUtils.randomNumeric(16)); given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) .when() - .post(DatabaseResource.BASE_PATH, "database") + .body(json) + .post(DatabaseResource.BASE_PATH, keyspaceId.asInternal()) .then() .statusCode(200); } + + @Test + public void error() { + given() + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) + .contentType(ContentType.JSON) + .when() + .post(DatabaseResource.BASE_PATH, keyspaceId.asInternal()) + .then() + .statusCode(400) + .body("errors[0].message", is("Request invalid: must not be null.")); + } } } From 73ef60f28baca26dc7e11324dd258df865f3ddfd Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 21:27:21 -0500 Subject: [PATCH 06/10] Fixed invalid integration test --- .../sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index 41c53d6898..fef76eca4d 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -45,8 +45,8 @@ public void happyPath() { given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) - .when() .body(json) + .when() .post(DatabaseResource.BASE_PATH, keyspaceId.asInternal()) .then() .statusCode(200); From 696cbf65e0e09328e5fd79ffe7c0448cb33e513e Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 21:36:00 -0500 Subject: [PATCH 07/10] Check for status code --- .../sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index fef76eca4d..e5a23517bc 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -2,7 +2,6 @@ import static io.restassured.RestAssured.given; import static io.stargate.sgv2.common.IntegrationTestUtils.getAuthToken; -import static org.hamcrest.Matchers.is; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; @@ -60,8 +59,7 @@ public void error() { .when() .post(DatabaseResource.BASE_PATH, keyspaceId.asInternal()) .then() - .statusCode(400) - .body("errors[0].message", is("Request invalid: must not be null.")); + .statusCode(400); } } } From c7878ba323c1e86fdeb43318a8f69e4e82cb7d83 Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 21:47:49 -0500 Subject: [PATCH 08/10] Changed the properties for native profile --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 52bedda22c..a073327761 100644 --- a/pom.xml +++ b/pom.xml @@ -176,7 +176,14 @@ native docsapi-native 6G + datastax/dse-server + 6.8.28 + stargateio/coordinator-dse-68 v${stargate.version} + dse-${stargate.int-test.cassandra.image-tag}-cluster + 6.8 + true + true From 78c9d283660e5ed788f1912f1893f6ebe116fc79 Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 22:21:01 -0500 Subject: [PATCH 09/10] Checking in to make native test rerun. --- .../sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index e5a23517bc..2e51a6d80a 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -41,6 +41,7 @@ public void happyPath() { } """, "col" + RandomStringUtils.randomNumeric(16)); + System.out.println("setting json"); given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) From d8d1acd95b5ac03df1797712a03955c08b5deaba Mon Sep 17 00:00:00 2001 From: Mahesh Rajamani Date: Thu, 22 Dec 2022 22:38:43 -0500 Subject: [PATCH 10/10] Checking in to make native test rerun. --- .../docsapi/api/v3/DatabaseResourceIntegrationTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java index 2e51a6d80a..2aed39e35c 100644 --- a/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java +++ b/src/test/java/io/stargate/sgv3/docsapi/api/v3/DatabaseResourceIntegrationTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; @QuarkusIntegrationTest @QuarkusTestResource(StargateTestResource.class) @@ -41,7 +42,6 @@ public void happyPath() { } """, "col" + RandomStringUtils.randomNumeric(16)); - System.out.println("setting json"); given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken()) .contentType(ContentType.JSON) @@ -53,6 +53,11 @@ public void happyPath() { } @Test + @DisabledIfSystemProperty( + named = "testing.package.type", + matches = "native", + disabledReason = + "[V2 exception mappers map to ApiError which is not registered for refection](https://github.com/riptano/sgv3-docsapi/issues/8)") public void error() { given() .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, getAuthToken())