From f58665af6100f8e1f23f96d6866507b7da43474f Mon Sep 17 00:00:00 2001 From: Josh Liburdi Date: Mon, 5 Aug 2024 09:09:33 -0700 Subject: [PATCH] fix(build/config): Self-Referential Jsonnet Function (#225) --- build/config/substation.libsonnet | 4 ++-- .../enrich/kvstore_csv/config.jsonnet | 22 +++++++++++++++++++ .../transform/enrich/kvstore_csv/data.jsonl | 1 + .../transform/enrich/kvstore_csv/kv.csv | 4 ++++ .../enrich/kvstore_json/config.jsonnet | 17 ++++++++++++++ .../transform/enrich/kvstore_json/data.jsonl | 1 + .../transform/enrich/kvstore_json/kv.json | 1 + .../enrich/kvstore_set_add/config.jsonnet | 2 +- 8 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 examples/config/transform/enrich/kvstore_csv/config.jsonnet create mode 100644 examples/config/transform/enrich/kvstore_csv/data.jsonl create mode 100644 examples/config/transform/enrich/kvstore_csv/kv.csv create mode 100644 examples/config/transform/enrich/kvstore_json/config.jsonnet create mode 100644 examples/config/transform/enrich/kvstore_json/data.jsonl create mode 100644 examples/config/transform/enrich/kvstore_json/kv.json diff --git a/build/config/substation.libsonnet b/build/config/substation.libsonnet index ec239ed5..7de38134 100644 --- a/build/config/substation.libsonnet +++ b/build/config/substation.libsonnet @@ -465,8 +465,8 @@ iget: $.transform.enrich.kv_store.item.get, iset: $.transform.enrich.kv_store.item.set, item: { - get: $.transform.enrich.kv_store.item.get, - set: $.transform.enrich.kv_store.item.set, + get: $.transform.enrich.kv_store.get, + set: $.transform.enrich.kv_store.set, }, // In future releases this will also be `set.add`. sadd(settings={}): { diff --git a/examples/config/transform/enrich/kvstore_csv/config.jsonnet b/examples/config/transform/enrich/kvstore_csv/config.jsonnet new file mode 100644 index 00000000..f648ef54 --- /dev/null +++ b/examples/config/transform/enrich/kvstore_csv/config.jsonnet @@ -0,0 +1,22 @@ +// This example shows how to use the `enrich_kv_store_item_get` transform +// to lookup data in a KV store backed by a CSV file. +local sub = import '../../../../../build/config/substation.libsonnet'; + +// This CSV file must be local to the Substation app. Absolute paths are +// recommended. Files accessible over HTTPS and hosted in AWS S3 also work. +// +// The `column` parameter is required and specifies the column in the CSV file +// that will be used to lookup the key in the KV store. +local kv = sub.kv_store.csv_file({ file: 'kv.csv', column: 'product' }); + +{ + transforms: [ + // The CSV file KV store returns the entire row minus the key column. + // For example, this returns {"price":"9.99","calories":"500"} for "churro". + sub.tf.enrich.kv_store.item.get({ + object: { source_key: 'product', target_key: 'info'}, + kv_store: kv, + }), + sub.tf.send.stdout(), + ], +} diff --git a/examples/config/transform/enrich/kvstore_csv/data.jsonl b/examples/config/transform/enrich/kvstore_csv/data.jsonl new file mode 100644 index 00000000..19a662e7 --- /dev/null +++ b/examples/config/transform/enrich/kvstore_csv/data.jsonl @@ -0,0 +1 @@ +{"product":"churro"} diff --git a/examples/config/transform/enrich/kvstore_csv/kv.csv b/examples/config/transform/enrich/kvstore_csv/kv.csv new file mode 100644 index 00000000..d52fdd05 --- /dev/null +++ b/examples/config/transform/enrich/kvstore_csv/kv.csv @@ -0,0 +1,4 @@ +product,price,calories +churro,9.99,500 +donut,1.99,300 +eclair,2.99,400 diff --git a/examples/config/transform/enrich/kvstore_json/config.jsonnet b/examples/config/transform/enrich/kvstore_json/config.jsonnet new file mode 100644 index 00000000..55335dc6 --- /dev/null +++ b/examples/config/transform/enrich/kvstore_json/config.jsonnet @@ -0,0 +1,17 @@ +// This example shows how to use the `enrich_kv_store_item_get` transform +// to lookup data in a KV store backed by a JSON file. +local sub = import '../../../../../build/config/substation.libsonnet'; + +// This JSON file must be local to the Substation app. Absolute paths are +// recommended. Files accessible over HTTPS and hosted in AWS S3 also work. +local kv = sub.kv_store.json_file({ file: 'kv.json' }); + +{ + transforms: [ + sub.tf.enrich.kv_store.item.get({ + object: { source_key: 'product', target_key: 'price'}, + kv_store: kv, + }), + sub.tf.send.stdout(), + ], +} diff --git a/examples/config/transform/enrich/kvstore_json/data.jsonl b/examples/config/transform/enrich/kvstore_json/data.jsonl new file mode 100644 index 00000000..19a662e7 --- /dev/null +++ b/examples/config/transform/enrich/kvstore_json/data.jsonl @@ -0,0 +1 @@ +{"product":"churro"} diff --git a/examples/config/transform/enrich/kvstore_json/kv.json b/examples/config/transform/enrich/kvstore_json/kv.json new file mode 100644 index 00000000..7d22510e --- /dev/null +++ b/examples/config/transform/enrich/kvstore_json/kv.json @@ -0,0 +1 @@ +{"churro":9.99} diff --git a/examples/config/transform/enrich/kvstore_set_add/config.jsonnet b/examples/config/transform/enrich/kvstore_set_add/config.jsonnet index 7fb8c133..01910d05 100644 --- a/examples/config/transform/enrich/kvstore_set_add/config.jsonnet +++ b/examples/config/transform/enrich/kvstore_set_add/config.jsonnet @@ -10,7 +10,7 @@ local mem = sub.kv_store.memory(); transforms: [ // Each order is stored in memory indexed by the customer's email // address and printed to stdout. Only unique orders are stored. - sub.tf.enrich.kv_store.set.add({ + sub.tf.enrich.kv_store.sadd({ object: { source_key: 'customer', target_key: 'order'}, kv_store: mem, ttl_offset: '10s',