Skip to content

Commit

Permalink
Merge pull request #152 from JordonPhillips/pagination
Browse files Browse the repository at this point in the history 
Allow using path expressions in paginator outputs
  • Loading branch information
@JordonPhillips
JordonPhillips authored Sep 9, 2019
2 parents fc13869 + 25f027d commit 1a28e83
Show file tree
Hide file tree
Showing 5 changed files with 284 additions and 30 deletions.
109 changes: 107 additions & 2 deletions docs/source/spec/core.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3883,7 +3883,7 @@ The ``paginated`` trait is an object that contains the following properties:
the service that contains the operation.
* - outputToken
- ``string``
- The name of the operation output member that contains an optional
- The path to the operation output member that contains an optional
continuation token. When this value is present in operation output,
it indicates that there are more results to retrieve. To get the next
page of results, the client passes the received output continuation
Expand All @@ -3896,7 +3896,7 @@ The ``paginated`` trait is an object that contains the following properties:
the service that contains the operation.
* - items
- ``string``
- The name of a top-level output member of the operation that contains
- The path to an output member of the operation that contains
the data that is being paginated across many responses. The named
output member, if specified, MUST target a list or map.
* - pageSize
Expand Down Expand Up @@ -4045,6 +4045,111 @@ settings from a service.
}
}

The values for ``outputToken`` and ``items`` are paths. :dfn:`Paths` are a series of
identifiers separated by dots (``.``) where each identifier represents a
member name in a structure. The first member name MUST correspond to a member
of the output structure and each subsequent member name MUST correspond to a
member in the previously referenced structure. Paths MUST adhere to the
following ABNF.

.. productionlist:: smithy
path :`identifier` *("." `identifier`)
The following example defines a paginated operation which uses a result
wrapper where the output token and items are referenced by paths.

.. tabs::

.. code-tab:: smithy

namespace smithy.example

@readonly
@paginated(inputToken: "nextToken", outputToken: "result.nextToken",
pageSize: "maxResults", items: "result.foos")
operation GetFoos(GetFoosInput) -> GetFoosOutput

structure GetFoosInput {
maxResults: Integer,
nextToken: String
}

structure GetFoosOutput {
@required
result: ResultWrapper
}

structure ResultWrapper {
nextToken: String,

@required
foos: StringList,
}

list StringList {
member: String
}

.. code-tab:: json

{
"smithy": "0.4.0",
"smithy.example": {
"shapes": {
"GetFoos": {
"type": "operation",
"input" :"GetFoosInput",
"output": "GetFoosOutput",
"readonly": true,
"paginated": {
"inputToken": "nextToken",
"outputToken": "result.nextToken",
"pageSize": "maxResults",
"items": "result.foos"
}
},
"GetFoosInput": {
"type": "structure",
"members": {
"maxResults": {
"target": "Integer"
}
"nextToken": {
"target": "String"
}
}
},
"GetFoosOutput": {
"type": "structure",
"members": {
"result": {
"target": "ResultWrapper",
"required": true
}
}
},
"ResultWrapper": {
"type": "structure",
"members": {
"nextToken": {
"target": "String"
},
"foos": {
"target": "StringList",
"required": true
}
}
},
"StringList": {
"type": "list",
"member": {
"target": "String"
}
}
}
}
}


Pagination Behavior
```````````````````
Expand Down
113 changes: 85 additions & 28 deletions ...main/java/software/amazon/smithy/model/validation/validators/PaginatedTraitValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
package software.amazon.smithy.model.validation.validators;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.OperationIndex;
Expand All @@ -30,6 +32,7 @@
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.ShapeIndex;
import software.amazon.smithy.model.shapes.ShapeType;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.model.traits.PaginatedTrait;
import software.amazon.smithy.model.traits.Trait;
import software.amazon.smithy.model.validation.AbstractValidator;
Expand Down Expand Up @@ -59,6 +62,7 @@ public final class PaginatedTraitValidator extends AbstractValidator {
private static final Set<ShapeType> ITEM_SHAPES = SetUtils.of(ShapeType.LIST, ShapeType.MAP);
private static final Set<ShapeType> PAGE_SHAPES = SetUtils.of(ShapeType.INTEGER);
private static final Set<ShapeType> STRING_SET = SetUtils.of(ShapeType.STRING);
private static final Pattern PATH_PATTERN = Pattern.compile("\\.");

@Override
public List<ValidationEvent> validate(Model model) {
Expand Down Expand Up @@ -120,20 +124,26 @@ private List<ValidationEvent> validateMember(
PropertyValidator validator
) {
String prefix = service != null ? "When bound within the `" + service.getId() + "` service, " : "";
String memberName = validator.getMemberName(opIndex, operation, trait).orElse(null);
String memberPath = validator.getMemberPath(opIndex, operation, trait).orElse(null);

if (memberName == null) {
if (memberPath == null) {
return service != null && validator.isRequiredToBePresent()
? Collections.singletonList(error(operation, trait, String.format(
"%spaginated trait `%s` is not configured", prefix, validator.propertyName())))
: Collections.emptyList();
}

MemberShape member = validator.getMember(opIndex, operation, trait).orElse(null);
if (!validator.pathsAllowed() && memberPath.contains(".")) {
return Collections.singletonList(error(operation, trait, String.format(
"%spaginated trait `%s` does not allow path values", prefix, validator.propertyName()
)));
}

MemberShape member = validator.getMember(index, opIndex, operation, trait).orElse(null);
if (member == null) {
return Collections.singletonList(error(operation, trait, String.format(
"%spaginated trait `%s` targets a member `%s` that does not exist",
prefix, validator.propertyName(), memberName)));
prefix, validator.propertyName(), memberPath)));
}

List<ValidationEvent> events = new ArrayList<>();
Expand All @@ -152,6 +162,14 @@ private List<ValidationEvent> validateMember(
ValidationUtils.tickedList(validator.validTargets()))));
}

if (validator.pathsAllowed() && PATH_PATTERN.split(memberPath).length > 2) {
events.add(warning(operation, trait, String.format(
"%spaginated trait `%s` contains a path with more than two parts, which can make your API "
+ "cumbersome to use",
prefix, validator.propertyName()
)));
}

return events;
}

Expand All @@ -164,9 +182,54 @@ private abstract static class PropertyValidator {

abstract Set<ShapeType> validTargets();

abstract Optional<String> getMemberName(OperationIndex index, OperationShape operation, PaginatedTrait trait);
abstract Optional<String> getMemberPath(OperationIndex opIndex, OperationShape operation, PaginatedTrait trait);

abstract Optional<MemberShape> getMember(
ShapeIndex index, OperationIndex opIndex, OperationShape operation, PaginatedTrait trait
);

boolean pathsAllowed() {
return false;
}
}

private abstract static class OutputPropertyValidator extends PropertyValidator {

@Override
boolean pathsAllowed() {
return true;
}

Optional<MemberShape> getMember(
ShapeIndex index, OperationIndex opIndex, OperationShape operation, PaginatedTrait trait
) {
// Split up the path expression into a list of member names
List<String> memberNames = getMemberPath(opIndex, operation, trait)
.map(value -> Arrays.asList(PATH_PATTERN.split(value)))
.orElse(Collections.emptyList());
if (memberNames.isEmpty()) {
return Optional.empty();
}
Optional<StructureShape> outputShape = opIndex.getOutput(operation);
if (!outputShape.isPresent()) {
return Optional.empty();
}

// Grab the first member from the output shape.
Optional<MemberShape> memberShape = outputShape.get().getMember(memberNames.get(0));

// For each member name in the path except the first, try to find that member in the previous structure
for (String memberName: memberNames.subList(1, memberNames.size())) {
if (!memberShape.isPresent()) {
return Optional.empty();
}
memberShape = index.getShape(memberShape.get().getTarget())
.flatMap(Shape::asStructureShape)
.flatMap(target -> target.getMember(memberName));
}
return memberShape;
}

abstract Optional<MemberShape> getMember(OperationIndex index, OperationShape operation, PaginatedTrait trait);
}

private static final class InputTokenValidator extends PropertyValidator {
Expand All @@ -186,17 +249,19 @@ Set<ShapeType> validTargets() {
return STRING_SET;
}

Optional<String> getMemberName(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
Optional<String> getMemberPath(OperationIndex opIndex, OperationShape operation, PaginatedTrait trait) {
return trait.getInputToken();
}

Optional<MemberShape> getMember(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
return getMemberName(index, operation, trait)
.flatMap(memberName -> index.getInput(operation).flatMap(input -> input.getMember(memberName)));
Optional<MemberShape> getMember(
ShapeIndex index, OperationIndex opIndex, OperationShape operation, PaginatedTrait trait
) {
return getMemberPath(opIndex, operation, trait)
.flatMap(memberName -> opIndex.getInput(operation).flatMap(input -> input.getMember(memberName)));
}
}

private static final class OutputTokenValidator extends PropertyValidator {
private static final class OutputTokenValidator extends OutputPropertyValidator {
boolean mustBeOptional() {
return true;
}
Expand All @@ -213,14 +278,9 @@ Set<ShapeType> validTargets() {
return STRING_SET;
}

Optional<String> getMemberName(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
Optional<String> getMemberPath(OperationIndex opIndex, OperationShape operation, PaginatedTrait trait) {
return trait.getOutputToken();
}

Optional<MemberShape> getMember(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
return getMemberName(index, operation, trait)
.flatMap(memberName -> index.getOutput(operation).flatMap(output -> output.getMember(memberName)));
}
}

private static final class PageSizeValidator extends PropertyValidator {
Expand All @@ -240,17 +300,19 @@ Set<ShapeType> validTargets() {
return PAGE_SHAPES;
}

Optional<String> getMemberName(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
Optional<String> getMemberPath(OperationIndex opIndex, OperationShape operation, PaginatedTrait trait) {
return trait.getPageSize();
}

Optional<MemberShape> getMember(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
return getMemberName(index, operation, trait)
.flatMap(memberName -> index.getInput(operation).flatMap(input -> input.getMember(memberName)));
Optional<MemberShape> getMember(
ShapeIndex index, OperationIndex opIndex, OperationShape operation, PaginatedTrait trait
) {
return getMemberPath(opIndex, operation, trait)
.flatMap(memberName -> opIndex.getInput(operation).flatMap(input -> input.getMember(memberName)));
}
}

private static final class ItemValidator extends PropertyValidator {
private static final class ItemValidator extends OutputPropertyValidator {
boolean mustBeOptional() {
return false;
}
Expand All @@ -267,13 +329,8 @@ Set<ShapeType> validTargets() {
return ITEM_SHAPES;
}

Optional<String> getMemberName(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
Optional<String> getMemberPath(OperationIndex opIndex, OperationShape operation, PaginatedTrait trait) {
return trait.getItems();
}

Optional<MemberShape> getMember(OperationIndex index, OperationShape operation, PaginatedTrait trait) {
return getMemberName(index, operation, trait)
.flatMap(memberName -> index.getOutput(operation).flatMap(output -> output.getMember(memberName)));
}
}
}
31 changes: 31 additions & 0 deletions smithy-model/src/test/java/software/amazon/smithy/model/traits/PaginatedTraitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,35 @@ public void mergesWithOtherTrait() {
assertThat(trait3.getPageSize(), equalTo(Optional.of("bar")));
assertThat(trait3.getItems(), equalTo(Optional.of("baz")));
}

@Test
public void allowsNestedOutputToken() {
TraitFactory provider = TraitFactory.createServiceFactory();
Node node = Node.objectNode()
.withMember("inputToken", Node.from("inputToken"))
.withMember("outputToken", Node.from("result.outputToken"));
Optional<Trait> trait = provider.createTrait(
ShapeId.from("smithy.api#paginated"), ShapeId.from("ns.qux#foo"), node);

assertThat(trait.isPresent(), is(true));
assertThat(trait.get(), instanceOf(PaginatedTrait.class));
PaginatedTrait paginatedTrait = (PaginatedTrait) trait.get();
assertThat(paginatedTrait.getOutputToken(), equalTo(Optional.of("result.outputToken")));
}

@Test
public void allowsNestedOutputItems() {
TraitFactory provider = TraitFactory.createServiceFactory();
Node node = Node.objectNode()
.withMember("items", Node.from("result.items"))
.withMember("inputToken", Node.from("inputToken"))
.withMember("outputToken", Node.from("outputToken"));
Optional<Trait> trait = provider.createTrait(
ShapeId.from("smithy.api#paginated"), ShapeId.from("ns.qux#foo"), node);

assertThat(trait.isPresent(), is(true));
assertThat(trait.get(), instanceOf(PaginatedTrait.class));
PaginatedTrait paginatedTrait = (PaginatedTrait) trait.get();
assertThat(paginatedTrait.getItems(), equalTo(Optional.of("result.items")));
}
}
4 changes: 4 additions & 0 deletions .../resources/software/amazon/smithy/model/errorfiles/validators/paginated-trait-test.errors
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,7 @@
[ERROR] ns.foo#UnresolvedOutput$nextToken: member shape targets an unresolved shape `ns.foo#Missing` | Target
[ERROR] ns.foo#Invalid9: When bound within the `ns.foo#Service` service, paginated trait `inputToken` is not configured | PaginatedTrait
[ERROR] ns.foo#Invalid9: When bound within the `ns.foo#Service` service, paginated trait `outputToken` is not configured | PaginatedTrait
[WARNING] ns.foo#DeeplyNestedOutputOperation: paginated trait `items` contains a path with more than two parts, which can make your API cumbersome to use | PaginatedTrait
[WARNING] ns.foo#DeeplyNestedOutputOperation: paginated trait `outputToken` contains a path with more than two parts, which can make your API cumbersome to use | PaginatedTrait
[ERROR] ns.foo#InvalidNestedInput: paginated trait `inputToken` does not allow path values | PaginatedTrait
[ERROR] ns.foo#InvalidNestedInput: paginated trait `pageSize` does not allow path values | PaginatedTrait
Loading

0 comments on commit 1a28e83

Please sign in to comment.