-
Notifications
You must be signed in to change notification settings - Fork 479
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Added support for DynamoDBTimeWindowEvent, KinesisTimeWindowEvent and StreamsEventResponse (for KinesisEvent). #1559
Changes from all commits
e4a46ba
9e0be81
2dd31a1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Text.Json; | ||
using System.Text.Json.Serialization; | ||
|
||
namespace Amazon.Lambda.DynamoDBEvents.Converters | ||
{ | ||
public class DictionaryLongToStringJsonConverter : JsonConverter<Dictionary<string, string>> | ||
{ | ||
public override Dictionary<string, string> Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) | ||
{ | ||
if (reader.TokenType != JsonTokenType.StartObject) | ||
{ | ||
throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported."); | ||
} | ||
|
||
var dictionary = new Dictionary<string, string>(); | ||
|
||
while (reader.Read()) | ||
{ | ||
if (reader.TokenType == JsonTokenType.EndObject) | ||
{ | ||
return dictionary; | ||
} | ||
|
||
// Get the key. | ||
if (reader.TokenType != JsonTokenType.PropertyName) | ||
{ | ||
throw new JsonException("JsonTokenType was not PropertyName."); | ||
} | ||
|
||
var propertyName = reader.GetString(); | ||
|
||
if (string.IsNullOrWhiteSpace(propertyName)) | ||
{ | ||
throw new JsonException("Failed to get property name."); | ||
} | ||
|
||
// Get the value. | ||
reader.Read(); | ||
var keyValue = ExtractValue(ref reader, options); | ||
dictionary.Add(propertyName, keyValue); | ||
} | ||
|
||
return dictionary; | ||
} | ||
|
||
public override void Write(Utf8JsonWriter writer, Dictionary<string, string> value, JsonSerializerOptions options) | ||
{ | ||
// Use the built-in serializer, because it can handle dictionaries with string keys. | ||
JsonSerializer.Serialize(writer, value, options); | ||
} | ||
|
||
private string ExtractValue(ref Utf8JsonReader reader, JsonSerializerOptions options) | ||
{ | ||
switch (reader.TokenType) | ||
{ | ||
case JsonTokenType.Number: | ||
if (reader.TryGetInt64(out var result)) | ||
{ | ||
return result.ToString(); | ||
} | ||
throw new JsonException($"Unable to convert '{reader.TokenType}' to long value."); | ||
case JsonTokenType.String: // If it is string, then use as it is. | ||
return reader.GetString(); | ||
default: | ||
throw new JsonException($"'{reader.TokenType}' is not supported."); | ||
} | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
namespace Amazon.Lambda.DynamoDBEvents | ||
ashishdhingra marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
ashishdhingra marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
#if NETCOREAPP3_1_OR_GREATER | ||
using Amazon.Lambda.DynamoDBEvents.Converters; | ||
using System.Text.Json.Serialization; | ||
#endif | ||
|
||
/// <summary> | ||
/// Represents an Amazon DynamodDB event when using time windows. | ||
/// https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html#services-ddb-windows | ||
/// </summary> | ||
public class DynamoDBTimeWindowEvent : DynamoDBEvent | ||
{ | ||
/// <summary> | ||
/// Time window for the records in the event. | ||
/// </summary> | ||
public TimeWindow Window { get; set; } | ||
|
||
/// <summary> | ||
/// State being built up to this invoke in the time window. | ||
/// </summary> | ||
#if NETCOREAPP3_1_OR_GREATER | ||
[JsonConverter(typeof(DictionaryLongToStringJsonConverter))] | ||
#endif | ||
public Dictionary<string, string> State { get; set; } | ||
|
||
/// <summary> | ||
/// Shard Id of the records. | ||
/// </summary> | ||
public string ShardId { get; set; } | ||
|
||
/// <summary> | ||
/// DynamoDB stream Arn. | ||
/// </summary> | ||
public string EventSourceArn { get; set; } | ||
|
||
/// <summary> | ||
/// Set to true for the last invoke of the time window. Subsequent invoke will start a new time window along with a fresh state. | ||
/// </summary> | ||
public bool? IsFinalInvokeForWindow { get; set; } | ||
|
||
/// <summary> | ||
/// Set to true if window is terminated prematurely. Subsequent invoke will continue the same window with a fresh state. | ||
/// </summary> | ||
public bool? IsWindowTerminatedEarly { get; set; } | ||
|
||
/// <summary> | ||
/// Time window for the records in the event. | ||
/// </summary> | ||
public class TimeWindow | ||
{ | ||
/// <summary> | ||
/// Window start instant. | ||
/// </summary> | ||
public DateTime Start { get; set; } | ||
|
||
/// <summary> | ||
/// Window end instant. | ||
/// </summary> | ||
public DateTime End { get; set; } | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
namespace Amazon.Lambda.DynamoDBEvents | ||
ashishdhingra marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Runtime.Serialization; | ||
ashishdhingra marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
#if NETCOREAPP3_1_OR_GREATER | ||
using Amazon.Lambda.DynamoDBEvents.Converters; | ||
using System.Text.Json.Serialization; | ||
#endif | ||
|
||
/// <summary> | ||
/// Response type to return a new state for the time window and to report batch item failures. | ||
/// </summary> | ||
[DataContract] | ||
public class DynamoDBTimeWindowResponse | ||
{ | ||
/// <summary> | ||
/// New state after processing a batch of records. | ||
/// </summary> | ||
[DataMember(Name = "state")] | ||
#if NETCOREAPP3_1_OR_GREATER | ||
[System.Text.Json.Serialization.JsonPropertyName("state")] | ||
[JsonConverter(typeof(DictionaryLongToStringJsonConverter))] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This can be combined with the previous There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have changed it to |
||
#endif | ||
public Dictionary<String, String> State { get; set; } | ||
|
||
/// <summary> | ||
/// A list of records which failed processing. | ||
/// Returning the first record which failed would retry all remaining records from the batch. | ||
/// </summary> | ||
[DataMember(Name = "batchItemFailures")] | ||
#if NETCOREAPP3_1_OR_GREATER | ||
[System.Text.Json.Serialization.JsonPropertyName("batchItemFailures")] | ||
#endif | ||
public IList<BatchItemFailure> BatchItemFailures { get; set; } | ||
|
||
/// <summary> | ||
/// Class representing the individual record which failed processing. | ||
/// </summary> | ||
[DataContract] | ||
public class BatchItemFailure | ||
{ | ||
/// <summary> | ||
/// Sequence number of the record which failed processing. | ||
/// </summary> | ||
[DataMember(Name = "itemIdentifier")] | ||
#if NETCOREAPP3_1_OR_GREATER | ||
[System.Text.Json.Serialization.JsonPropertyName("itemIdentifier")] | ||
#endif | ||
public string ItemIdentifier { get; set; } | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Text.Json; | ||
using System.Text.Json.Serialization; | ||
|
||
namespace Amazon.Lambda.KinesisEvents.Converters | ||
{ | ||
public class DictionaryLongToStringJsonConverter : JsonConverter<Dictionary<string, string>> | ||
{ | ||
public override Dictionary<string, string> Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) | ||
{ | ||
if (reader.TokenType != JsonTokenType.StartObject) | ||
{ | ||
throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported."); | ||
} | ||
|
||
var dictionary = new Dictionary<string, string>(); | ||
|
||
while (reader.Read()) | ||
{ | ||
if (reader.TokenType == JsonTokenType.EndObject) | ||
{ | ||
return dictionary; | ||
} | ||
|
||
// Get the key. | ||
if (reader.TokenType != JsonTokenType.PropertyName) | ||
{ | ||
throw new JsonException("JsonTokenType was not PropertyName."); | ||
} | ||
|
||
var propertyName = reader.GetString(); | ||
|
||
if (string.IsNullOrWhiteSpace(propertyName)) | ||
{ | ||
throw new JsonException("Failed to get property name."); | ||
} | ||
|
||
// Get the value. | ||
reader.Read(); | ||
var keyValue = ExtractValue(ref reader, options); | ||
dictionary.Add(propertyName, keyValue); | ||
} | ||
|
||
return dictionary; | ||
} | ||
|
||
public override void Write(Utf8JsonWriter writer, Dictionary<string, string> value, JsonSerializerOptions options) | ||
{ | ||
// Use the built-in serializer, because it can handle dictionaries with string keys. | ||
JsonSerializer.Serialize(writer, value, options); | ||
} | ||
|
||
private string ExtractValue(ref Utf8JsonReader reader, JsonSerializerOptions options) | ||
{ | ||
switch (reader.TokenType) | ||
{ | ||
case JsonTokenType.Number: | ||
if (reader.TryGetInt64(out var result)) | ||
{ | ||
return result.ToString(); | ||
} | ||
throw new JsonException($"Unable to convert '{reader.TokenType}' to long value."); | ||
case JsonTokenType.String: // If it is string, then use as it is. | ||
return reader.GetString(); | ||
default: | ||
throw new JsonException($"'{reader.TokenType}' is not supported."); | ||
} | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Wouldn't this write the value as a string but given the context of this converter shouldn't it be written as a long?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, it would. I still haven’t received response from Lambda team. My hunch is that the reason there is a note to use
Map<string, string>
for Java is that sometimes the state might contain string values. In such case, we would not be sure on whether to convert it to long or string. That’s the reason I’m handling token type string as well in ExtractValue. Please advise.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As discussed, got response from Lambda team that
State
field could have values represented as strings and in some cases long.