Skip to content

Commit

Permalink
docs: Update Kinesis Tap README (#201)
Browse files Browse the repository at this point in the history
* docs: Update Kinesis Tap README

* style: jsonnetfmt
  • Loading branch information
jshlbrd committed Jul 9, 2024
1 parent 0308c74 commit 32298b1
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 8 deletions.
5 changes: 4 additions & 1 deletion cmd/development/kinesis-tap/substation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,12 @@ DEBU[0004] Flushed Substation pipeline.

## Build

Download, configure, and build the `kinesis-tap` binary with these commands:

```
git clone github.com/brexhq/substation && \
git clone https://github.com/brexhq/substation.git && \
cd substation/cmd/development/kinesis-tap/substation && \
jsonnet config.jsonnet > config.json && \
go build .
```

Expand Down
2 changes: 1 addition & 1 deletion cmd/development/kinesis-tap/substation/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ local sub = import '../../../../build/config/substation.libsonnet';
{
transforms: [
sub.tf.send.stdout(),
]
],
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ local sub = import '../../../../../build/config/substation.libsonnet';
transforms: [
sub.tf.send.aws.s3({
// Glacier Instant Retrieval charges a minimum of 128KB per object, otherwise
// the other values are set to impossibly high values to ensure all events are
// the other values are set to impossibly high values to ensure all events are
// written to the same file.
batch: { size: 128 * 1000, count: 1000 * 1000, duration: '60m' },
bucket_name: 'substation',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ local dest = { type: 'aws_cloudwatch_embedded_metrics' };
// If the transform is configured first, then the metric reflects
// the sum of bytes received by Substation.
sub.transform.utility.metric.bytes({ metric: { name: 'BytesReceived', attributes: attr, destination: dest } }),
// This inserts a value into the object so that the message size increases.
sub.transform.object.insert({obj: {target_key: '_'}, value: 1}),
// This inserts a value into the object so that the message size increases.
sub.transform.object.insert({ obj: { target_key: '_' }, value: 1 }),
// If the transform is configured last, then the metric reflects
// the sum of bytes transformed by Substation.
sub.transform.utility.metric.bytes({ metric: { name: 'BytesTransformed', attributes: attr, destination: dest } }),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
// time and determining if the difference is less than a threshold:
// - Success: current time - timestamp < threshold
// - Failure: current time - timestamp >= threshold
//
//
// The transform emits two metrics that describe success and failure, annotated
// in the `FreshnessType` attribute.
local sub = import '../../../../../build/config/substation.libsonnet';
Expand All @@ -15,10 +15,10 @@ local dest = { type: 'aws_cloudwatch_embedded_metrics' };

{
transforms: [
sub.transform.utility.metric.freshness({
sub.transform.utility.metric.freshness({
threshold: '5s', // Amount of time spent in the system before considered stale.
object: { source_key: 'timestamp' }, // Used as the reference to determine freshness.
metric: { name: 'MessageFreshness', attributes: attr, destination: dest }
metric: { name: 'MessageFreshness', attributes: attr, destination: dest },
}),
],
}

0 comments on commit 32298b1

Please sign in to comment.