From ab1a4ff7b1f7b9a7e216e3794bbbf325206ef927 Mon Sep 17 00:00:00 2001 From: Josh Liburdi Date: Mon, 10 Jun 2024 09:04:18 -0700 Subject: [PATCH] feat(transform): Add ID Settings Field (#181) --- build/config/substation.libsonnet | 379 ++++++++++++------ examples/config/config.jsonnet | 2 +- transform/aggregate.go | 2 + transform/aggregate_from_array.go | 12 +- transform/aggregate_from_string.go | 8 +- transform/aggregate_to_array.go | 14 +- transform/aggregate_to_string.go | 12 +- transform/array_join.go | 11 +- transform/array_zip.go | 11 +- transform/enrich.go | 1 + transform/enrich_aws_dynamodb.go | 15 +- transform/enrich_aws_lambda.go | 17 +- transform/enrich_dns_domain_lookup.go | 16 +- transform/enrich_dns_ip_lookup.go | 16 +- transform/enrich_dns_txt_lookup.go | 16 +- transform/enrich_http_get.go | 19 +- transform/enrich_http_post.go | 19 +- transform/enrich_kv_store_get.go | 19 +- transform/enrich_kv_store_set.go | 25 +- transform/format.go | 5 +- transform/format_from_base64.go | 14 +- transform/format_from_gzip.go | 8 +- transform/format_from_pretty_print.go | 14 +- transform/format_to_base64.go | 10 +- transform/format_to_gzip.go | 8 +- transform/hash.go | 1 + transform/hash_md5.go | 8 +- transform/hash_sha256.go | 8 +- transform/meta_err.go | 16 +- transform/meta_for_each.go | 17 +- transform/meta_kv_store_lock.go | 23 +- transform/meta_metric_duration.go | 15 +- transform/meta_pipeline.go | 21 +- transform/meta_switch.go | 20 +- transform/network.go | 1 + transform/network_domain_registered_domain.go | 14 +- transform/network_domain_subdomain.go | 14 +- transform/network_domain_top_level_domain.go | 10 +- transform/number.go | 1 + transform/number_math_addition.go | 12 +- transform/number_math_division.go | 12 +- transform/number_math_multiplication.go | 12 +- transform/number_math_subtraction.go | 12 +- transform/object_copy.go | 11 +- transform/object_delete.go | 11 +- transform/object_insert.go | 11 +- transform/object_jq.go | 18 +- transform/object_to_boolean.go | 11 +- transform/object_to_float.go | 9 +- transform/object_to_integer.go | 9 +- transform/object_to_string.go | 9 +- transform/object_to_unsigned_integer.go | 9 +- transform/send_aws_dynamodb.go | 21 +- transform/send_aws_kinesis_data_firehose.go | 19 +- transform/send_aws_kinesis_data_stream.go | 21 +- transform/send_aws_lambda.go | 21 +- transform/send_aws_s3.go | 21 +- transform/send_aws_sns.go | 19 +- transform/send_aws_sqs.go | 19 +- transform/send_file.go | 21 +- transform/send_http_post.go | 17 +- transform/send_stdout.go | 17 +- transform/string.go | 1 + transform/string_append.go | 11 +- transform/string_capture.go | 19 +- transform/string_replace.go | 11 +- transform/string_split.go | 13 +- transform/string_to_lower.go | 10 +- transform/string_to_snake.go | 10 +- transform/string_to_upper.go | 10 +- transform/string_uuid.go | 9 +- transform/time.go | 2 + transform/time_from_string.go | 12 +- transform/time_from_unix.go | 10 +- transform/time_from_unix_milli.go | 10 +- transform/time_now.go | 11 +- transform/time_to_string.go | 12 +- transform/time_to_unix.go | 10 +- transform/time_to_unix_milli.go | 10 +- transform/transform.go | 2 +- transform/utility_control.go | 11 +- transform/utility_delay.go | 12 +- transform/utility_drop.go | 10 +- transform/utility_err.go | 8 +- transform/utility_metric_bytes.go | 12 +- transform/utility_metric_count.go | 12 +- transform/utility_metric_freshness.go | 21 +- transform/utility_secret.go | 14 +- 88 files changed, 983 insertions(+), 474 deletions(-) diff --git a/build/config/substation.libsonnet b/build/config/substation.libsonnet index cea9182e..8184568b 100644 --- a/build/config/substation.libsonnet +++ b/build/config/substation.libsonnet @@ -250,42 +250,50 @@ from: { arr(settings={}): $.transform.aggregate.from.array(settings=settings), array(settings={}): { + local type = 'aggregate_from_array', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'aggregate_from_array', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, str(settings={}): $.transform.aggregate.from.string(settings=settings), string(settings={}): { + local type = 'aggregate_from_string', local default = { + id: $.helpers.id(type, settings), separator: null, }, - type: 'aggregate_from_string', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, to: { arr(settings={}): $.transform.aggregate.to.array(settings=settings), array(settings={}): { + local type = 'aggregate_to_array', local default = { + id: $.helpers.id(type, settings), object: $.config.object, batch: $.config.batch, }, - type: 'aggregate_to_array', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, str(settings={}): $.transform.aggregate.to.string(settings=settings), string(settings={}): { + local type = 'aggregate_to_string', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, separator: null, }, - type: 'aggregate_to_string', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -293,41 +301,33 @@ arr: $.transform.array, array: { join(settings={}): { + local type = 'array_join', local default = { + id: $.helpers.id(type, settings), object: $.config.object, separator: null, }, - type: 'array_join', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, - to: { - obj: $.transform.array.to.object, - object(settings={}): { - local default = { - object: $.config.object, - object_keys: null, - }, - - type: 'array_to_object', - settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), - }, - }, zip(settings={}): { + local type = 'array_zip', local default = { + id: $.helpers.id(type, settings), object: $.config.object, - as_object: false, - with_keys: null, }, - type: 'array_zip', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, enrich: { aws: { dynamodb(settings={}): { + local type = 'enrich_aws_dynamodb', local default = { + id: $.helpers.id(type, settings), object: $.config.object, aws: $.config.aws, retry: $.config.retry, @@ -339,18 +339,20 @@ scan_index_forward: false, }, - type: 'enrich_aws_dynamodb', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, lambda(settings={}): { + local type = 'enrich_aws_lambda', local default = { + id: $.helpers.id(type, settings), object: $.config.object, aws: $.config.aws, retry: $.config.retry, function_name: null, }, - type: 'enrich_aws_lambda', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -360,21 +362,24 @@ request: $.config.request, }, domain_lookup(settings={}): { - local default = $.transform.enrich.dns.default, + local type = 'enrich_dns_domain_lookup', + local default = $.transform.enrich.dns.default { id: $.helpers.id(type, settings) }, - type: 'enrich_dns_domain_lookup', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, ip_lookup(settings={}): { - local default = $.transform.enrich.dns.default, + local type = 'enrich_dns_ip_lookup', + local default = $.transform.enrich.dns.default { id: $.helpers.id(type, settings) }, - type: 'enrich_dns_ip_lookup', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, txt_lookup(settings={}): { - local default = $.transform.enrich.dns.default, + local type = 'enrich_dns_txt_lookup', + local default = $.transform.enrich.dns.default { id: $.helpers.id(type, settings) }, - type: 'enrich_dns_txt_lookup', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -386,15 +391,17 @@ headers: null, }, get(settings={}): { - local default = $.transform.enrich.http.default, + local type = 'enrich_http_get', + local default = $.transform.enrich.http.default { id: $.helpers.id(type, settings)}, - type: 'enrich_http_get', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, post(settings={}): { - local default = $.transform.enrich.http.default { body_key: null }, + local type = 'enrich_http_post', + local default = $.transform.enrich.http.default { body_key: null, id: $.helpers.id(type, settings) }, - type: 'enrich_http_post', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -406,15 +413,17 @@ close_kv_store: false, }, get(settings={}): { - local default = $.transform.enrich.kv_store.default, + local type = 'enrich_kv_store_get', + local default = $.transform.enrich.kv_store.default {id: $.helpers.id(type, settings)}, - type: 'enrich_kv_store_get', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, set(settings={}): { - local default = $.transform.enrich.kv_store.default { ttl_key: null, ttl_offset: '0s' }, + local type = 'enrich_kv_store_set', + local default = $.transform.enrich.kv_store.default { ttl_key: null, ttl_offset: '0s', id: $.helpers.id(type, settings) }, - type: 'enrich_kv_store_set', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -427,30 +436,44 @@ from: { b64(settings={}): $.transform.format.from.base64(settings=settings), base64(settings={}): { - local default = $.transform.format.default, + local type = 'format_from_base64', + local default = $.transform.format.default { id: $.helpers.id(type, settings) }, - type: 'format_from_base64', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, gz(settings={}): $.transform.format.from.gzip(settings=settings), gzip(settings={}): { - type: 'format_from_gzip', + local type = 'format_from_gzip', + local default = { id: $.helpers.id(type, settings) }, + + type: type, + settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, pretty_print(settings={}): { - type: 'format_from_pretty_print', + local type = 'format_from_pretty_print', + local default = { id: $.helpers.id(type, settings) }, + + type: type, + settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, to: { b64(settings={}): $.transform.format.to.base64(settings=settings), base64(settings={}): { - local default = $.transform.format.default, + local type = 'format_to_base64', + local default = $.transform.format.default { id: $.helpers.id(type, settings) }, - type: 'format_to_base64', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, gz(settings={}): $.transform.format.to.gzip(settings=settings), gzip(settings={}): { - type: 'format_to_gzip', + local type = 'format_to_gzip', + local default = { id: $.helpers.id(type, settings) }, + + type: type, + settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, }, @@ -459,15 +482,17 @@ object: $.config.object, }, md5(settings={}): { - local default = $.transform.hash.default, + local type = 'hash_md5', + local default = $.transform.hash.default { id: $.helpers.id(type, settings) }, - type: 'hash_md5', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, sha256(settings={}): { - local default = $.transform.hash.default, + local type = 'hash_sha256', + local default = $.transform.hash.default { id: $.helpers.id(type, settings) }, - type: 'hash_sha256', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -479,53 +504,66 @@ }, add(settings={}): $.transform.number.math.addition(settings=settings), addition(settings={}): { - local default = $.transform.number.math.default, + local type = 'number_math_addition', + local default = $.transform.number.math.default { id: $.helpers.id(type, settings) }, - type: 'number_math_addition', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, sub(settings={}): $.transform.number.math.subtraction(settings=settings), subtraction(settings={}): { - local default = $.transform.number.math.default, + local type = 'number_math_subtraction', + local default = $.transform.number.math.default { id: $.helpers.id(type, settings) }, - type: 'number_math_subtraction', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, mul(settings={}): $.transform.number.math.multiplication(settings=settings), multiplication(settings={}): { - local default = $.transform.number.math.default, + local type = 'number_math_multiplication', + local default = $.transform.number.math.default { id: $.helpers.id(type, settings) }, - type: 'number_math_multiplication', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, div(settings={}): $.transform.number.math.division(settings=settings), division(settings={}): { - local default = $.transform.number.math.default, + local type = 'number_math_division', + local default = $.transform.number.math.default { id: $.helpers.id(type, settings) }, - type: 'number_math_division', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, }, meta: { err(settings={}): { - local default = { transform: null, error_messages: null }, + local type = 'meta_err', + local default = { + id: $.helpers.id(type, settings), + transform: null, + error_messages: null, + }, - type: 'meta_err', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, for_each(settings={}): { + local type = 'meta_for_each', local default = { + id: $.helpers.id(type, settings), object: $.config.object, transform: null, }, - type: 'meta_for_each', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, kv_store: { lock(settings={}): { + local type = 'meta_kv_store_lock', local default = { + id: $.helpers.id(type, settings), object: $.config.object { ttl_key: null }, transform: null, kv_store: null, @@ -533,35 +571,43 @@ ttl_offset: '0s', }, - type: 'meta_kv_store_lock', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, metric: { duration(settings={}): { + local type = 'meta_metric_duration', local default = { + id: $.helpers.id(type, settings), metric: $.config.metric, transform: null, }, - type: 'meta_metric_duration', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, pipe(settings={}): $.transform.meta.pipeline(settings=settings), pipeline(settings={}): { + local type = 'meta_pipeline', local default = { + id: $.helpers.id(type, settings), object: $.config.object, transforms: null, }, - type: 'meta_pipeline', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, switch(settings={}): { - local default = { cases: null }, + local type = 'meta_switch', + local default = { + id: $.helpers.id(type, settings), + cases: null + }, - type: 'meta_switch', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -572,22 +618,25 @@ object: $.config.object, }, registered_domain(settings={}): { - local default = $.transform.network.domain.default, + local type = 'network_domain_registered_domain', + local default = $.transform.network.domain.default { id: $.helpers.id(type, settings) }, - type: 'network_domain_registered_domain', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, subdomain(settings={}): { - local default = $.transform.network.domain.default, + local type = 'network_domain_subdomain', + local default = $.transform.network.domain.default { id: $.helpers.id(type, settings) }, - type: 'network_domain_subdomain', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, tld(settings={}): $.transform.network.domain.top_level_domain(settings=settings), top_level_domain(settings={}): { - local default = $.transform.network.domain.default, + local type = 'network_domain_top_level_domain', + local default = $.transform.network.domain.default { id: $.helpers.id(type, settings) }, - type: 'network_domain_top_level_domain', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -599,63 +648,75 @@ }, cp(settings={}): $.transform.object.copy(settings=settings), copy(settings={}): { - local default = $.transform.object.default, + local type = 'object_copy', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_copy', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, del(settings={}): $.transform.object.delete(settings=settings), delete(settings={}): { - local default = $.transform.object.default, + local type = 'object_delete', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_delete', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, insert(settings={}): { - local default = $.transform.object.default, + local type = 'object_insert', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_insert', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, jq(settings={}): { - local default = { filter: null }, + local type = 'object_jq', + local default = { + id: $.helpers.id(type, settings), + filter: null + }, - type: 'object_jq', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, to: { bool(settings={}): $.transform.object.to.boolean(settings=settings), boolean(settings={}): { - local default = $.transform.object.default, + local type = 'object_to_boolean', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_to_boolean', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, float(settings={}): { - local default = $.transform.object.default, + local type = 'object_to_float', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_to_float', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, int(settings={}): $.transform.object.to.integer(settings=settings), integer(settings={}): { - local default = $.transform.object.default, + local type = 'object_to_integer', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_to_integer', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, str(settings={}): $.transform.object.to.string(settings=settings), string(settings={}): { - local default = $.transform.object.default, + local type = 'object_to_string', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_to_string', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, uint(settings={}): $.transform.object.to.unsigned_integer(settings=settings), unsigned_integer(settings={}): { - local default = $.transform.object.default, + local type = 'object_to_unsigned_integer', + local default = $.transform.object.default { id: $.helpers.id(type, settings) }, - type: 'object_to_unsigned_integer', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -663,7 +724,9 @@ send: { aws: { dynamodb(settings={}): { + local type = 'send_aws_dynamodb', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -676,12 +739,14 @@ aux_tforms: null, }), - type: 'send_aws_dynamodb', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, firehose(settings={}): $.transform.send.aws.kinesis_data_firehose(settings=settings), kinesis_data_firehose(settings={}): { + local type = 'send_aws_kinesis_data_firehose', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -694,11 +759,13 @@ aux_tforms: null, }), - type: 'send_aws_kinesis_data_firehose', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, kinesis_data_stream(settings={}): { + local type = 'send_aws_kinesis_data_stream', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -713,11 +780,13 @@ aux_tforms: null, }), - type: 'send_aws_kinesis_data_stream', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, lambda(settings={}): { + local type = 'send_aws_lambda', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -725,11 +794,13 @@ function_name: null, }, - type: 'send_aws_lambda', + type: type, settings: std.mergePatch(default, $.helpers.abbv(settings)), }, s3(settings={}): { + local type = 'send_aws_s3', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -743,11 +814,13 @@ aux_tforms: null, }), - type: 'send_aws_s3', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, sns(settings={}): { + local type = 'send_aws_sns', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -760,11 +833,13 @@ aux_tforms: null, }), - type: 'send_aws_sns', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, sqs(settings={}): { + local type = 'send_aws_sqs', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, aws: $.config.aws, @@ -777,12 +852,14 @@ aux_tforms: null, }), - type: 'send_aws_sqs', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, }, file(settings={}): { + local type = 'send_file', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, file_path: $.file_path, @@ -793,12 +870,14 @@ aux_tforms: null, }), - type: 'send_file', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, http: { post(settings={}): { + local type = 'send_http_post', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, url: null, @@ -812,12 +891,14 @@ hdr: null, }), - type: 'send_http_post', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, }, stdout(settings={}): { + local type = 'send_stdout', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, auxiliary_transforms: null, }, @@ -827,34 +908,40 @@ aux_tforms: null, }), - type: 'send_stdout', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, }, str: $.transform.string, string: { append(settings={}): { + local type = 'string_append', local default = { + id: $.helpers.id(type, settings), object: $.config.object, suffix: null, }, - type: 'string_append', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, capture(settings={}): { + local type = 'string_capture', local default = { + id: $.helpers.id(type, settings), object: $.config.object, pattern: null, count: 0, }, - type: 'string_capture', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, repl: $.transform.string.replace, replace(settings={}): { + local type = 'string_replace', local default = { + id: $.helpers.id(type, settings), object: $.config.object, pattern: null, replacement: null, @@ -866,11 +953,13 @@ repl: null, }), - type: 'string_replace', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, split(settings={}): { + local type = 'string_split', local default = { + id: $.helpers.id(type, settings), object: $.config.object, separator: null, }, @@ -880,7 +969,7 @@ sep: null, }), - type: 'string_split', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(s))), }, to: { @@ -888,30 +977,35 @@ object: $.config.object, }, lower(settings={}): { - local default = $.transform.string.to.default, + local type = 'string_to_lower', + local default = $.transform.string.to.default { id: $.helpers.id(type, settings) }, - type: 'string_to_lower', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, upper(settings={}): { - local default = $.transform.string.to.default, + local type = 'string_to_upper', + local default = $.transform.string.to.default { id: $.helpers.id(type, settings) }, - type: 'string_to_upper', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, snake(settings={}): { - local default = $.transform.string.to.default, + local type = 'string_to_snake', + local default = $.transform.string.to.default { id: $.helpers.id(type, settings) }, - type: 'string_to_snake', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, uuid(settings={}): { + local type = 'string_uuid', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'string_uuid', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -919,131 +1013,167 @@ from: { str(settings={}): $.transform.time.from.string(settings=settings), string(settings={}): { + local type = 'time_from_string', local default = { + id: $.helpers.id(type, settings), object: $.config.object, format: null, location: null, }, - type: 'time_from_string', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, unix(settings={}): { + local type = 'time_from_unix', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'time_from_unix', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, unix_milli(settings={}): { + local type = 'time_from_unix_milli', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'time_from_unix_milli', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, now(settings={}): { + local type = 'time_now', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'time_now', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, to: { str(settings={}): $.transform.time.to.string(settings=settings), string(settings={}): { + local type = 'time_to_string', local default = { + id: $.helpers.id(type, settings), object: $.config.object, format: null, location: null, }, - type: 'time_to_string', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, unix(settings={}): { + local type = 'time_to_unix', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'time_to_unix', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, unix_milli(settings={}): { + local type = 'time_to_unix_milli', local default = { + id: $.helpers.id(type, settings), object: $.config.object, }, - type: 'time_to_unix_milli', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, util: $.transform.utility, utility: { control(settings={}): { + local type = 'utility_control', local default = { + id: $.helpers.id(type, settings), batch: $.config.batch, }, - type: 'utility_control', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, delay(settings={}): { + local type = 'utility_delay', local default = { + id: $.helpers.id(type, settings), duration: null, }, - type: 'utility_delay', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, drop(settings={}): { - type: 'utility_drop', + local type = 'utility_delay', + local default = { + id: $.helpers.id(type, settings), + }, + + type: type, + settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, err(settings={}): { + local type = 'utility_err', local default = { + id: $.helpers.id(type, settings), message: null, }, - type: 'utility_err', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, metric: { bytes(settings={}): { + local type = 'utility_metric_bytes', local default = { + id: $.helpers.id(type, settings), metric: $.config.metric, }, - type: 'utility_metric_bytes', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, count(settings={}): { + local type = 'utility_metric_count', local default = { + id: $.helpers.id(type, settings), metric: $.config.metric, }, - type: 'utility_metric_count', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, freshness(settings={}): { + local type = 'utility_metric_freshness', local default = { + id: $.helpers.id(type, settings), threshold: null, metric: $.config.metric, object: $.config.object, }, - type: 'utility_metric_freshness', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, secret(settings={}): { - local default = { secret: null }, + local type = 'utility_secret', + local default = { + id: $.helpers.id(type, settings), + secret: null + }, - type: 'utility_secret', + type: type, settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))), }, }, @@ -1223,5 +1353,6 @@ trg: null, batch_key: if std.objectHas(s, 'btch') then s.batch else if std.objectHas(s, 'batch_key') then s.batch_key else null, }, + id(type, settings): std.join("-", [std.md5(type)[:8], std.md5(std.toString(settings))[:8]]) }, } diff --git a/examples/config/config.jsonnet b/examples/config/config.jsonnet index dd98afb7..783f2e58 100644 --- a/examples/config/config.jsonnet +++ b/examples/config/config.jsonnet @@ -4,7 +4,7 @@ local sub = import '../../build/config/substation.libsonnet'; // Substation application configs always contain an array named `transforms`. transforms: [ // Each transform function is defined in the `substation` library. - sub.transform.object.insert({ object: { target_key: 'a' }, value: 'z' }), + sub.transform.object.insert({ id: 'insert-z', object: { target_key: 'a' }, value: 'z' }), // Transform functions can be conditionally applied using the // `meta.switch` function. sub.transform.meta.switch({ cases: [ diff --git a/transform/aggregate.go b/transform/aggregate.go index e4d4d3be..f9883a28 100644 --- a/transform/aggregate.go +++ b/transform/aggregate.go @@ -10,6 +10,7 @@ import ( ) type aggregateArrayConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` } @@ -30,6 +31,7 @@ type aggregateStrConfig struct { // Separator is the string that is used to join and split data. Separator string `json:"separator"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` } diff --git a/transform/aggregate_from_array.go b/transform/aggregate_from_array.go index a0e11e0e..4097591d 100644 --- a/transform/aggregate_from_array.go +++ b/transform/aggregate_from_array.go @@ -13,7 +13,11 @@ import ( func newAggregateFromArray(_ context.Context, cfg config.Config) (*aggregateFromArray, error) { conf := aggregateArrayConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: aggregate_from_array: %v", err) + return nil, fmt.Errorf("transform aggregate_from_array: %v", err) + } + + if conf.ID == "" { + conf.ID = "aggregate_from_array" } tf := aggregateFromArray{ @@ -43,7 +47,7 @@ func (tf *aggregateFromArray) Transform(ctx context.Context, msg *message.Messag if tf.hasObjSrc { value = msg.GetValue(tf.conf.Object.SourceKey) if err := msg.DeleteValue(tf.conf.Object.SourceKey); err != nil { - return nil, err + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value = bytesToValue(msg.Data()) @@ -59,14 +63,14 @@ func (tf *aggregateFromArray) Transform(ctx context.Context, msg *message.Messag if tf.hasObjSrc { for key, val := range msg.GetValue("@this").Map() { if err := outMsg.SetValue(key, val.Value()); err != nil { - return nil, err + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } } if tf.hasObjTrg { if err := outMsg.SetValue(tf.conf.Object.TargetKey, res); err != nil { - return nil, err + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } output = append(output, outMsg) diff --git a/transform/aggregate_from_string.go b/transform/aggregate_from_string.go index 8716004d..6a529b00 100644 --- a/transform/aggregate_from_string.go +++ b/transform/aggregate_from_string.go @@ -12,11 +12,15 @@ import ( func newAggregateFromString(_ context.Context, cfg config.Config) (*aggregateFromString, error) { conf := aggregateStrConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: aggregate_from_string: %v", err) + return nil, fmt.Errorf("transform aggregate_from_string: %v", err) + } + + if conf.ID == "" { + conf.ID = "aggregate_from_string" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: aggregate_from_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := aggregateFromString{ diff --git a/transform/aggregate_to_array.go b/transform/aggregate_to_array.go index 4e302986..cd6e7546 100644 --- a/transform/aggregate_to_array.go +++ b/transform/aggregate_to_array.go @@ -14,7 +14,11 @@ import ( func newAggregateToArray(_ context.Context, cfg config.Config) (*aggregateToArray, error) { conf := aggregateArrayConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: aggregate_to_array: %v", err) + return nil, fmt.Errorf("transform aggregate_to_array: %v", err) + } + + if conf.ID == "" { + conf.ID = "aggregate_to_array" } tf := aggregateToArray{ @@ -28,7 +32,7 @@ func newAggregateToArray(_ context.Context, cfg config.Config) (*aggregateToArra Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: aggregate_to_array: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = *agg @@ -56,7 +60,7 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) outMsg := message.New() if tf.hasObjTrg { if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { - return nil, fmt.Errorf("transform: aggregate_to_array: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { outMsg.SetData(array) @@ -81,7 +85,7 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) outMsg := message.New() if tf.hasObjTrg { if err := outMsg.SetValue(tf.conf.Object.TargetKey, array); err != nil { - return nil, fmt.Errorf("transform: aggregate_to_array: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { outMsg.SetData(array) @@ -90,7 +94,7 @@ func (tf *aggregateToArray) Transform(ctx context.Context, msg *message.Message) // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_stdout: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{outMsg}, nil diff --git a/transform/aggregate_to_string.go b/transform/aggregate_to_string.go index a64e8b6b..7eb5f489 100644 --- a/transform/aggregate_to_string.go +++ b/transform/aggregate_to_string.go @@ -14,11 +14,15 @@ import ( func newAggregateToString(_ context.Context, cfg config.Config) (*aggregateToString, error) { conf := aggregateStrConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: aggregate_to_string: %v", err) + return nil, fmt.Errorf("transform aggregate_to_string: %v", err) + } + + if conf.ID == "" { + conf.ID = "aggregate_to_string" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: aggregate_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := aggregateToString{ @@ -32,7 +36,7 @@ func newAggregateToString(_ context.Context, cfg config.Config) (*aggregateToStr Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: aggregate_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -80,7 +84,7 @@ func (tf *aggregateToString) Transform(ctx context.Context, msg *message.Message // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_stdout: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{outMsg}, nil diff --git a/transform/array_join.go b/transform/array_join.go index 977b9add..362b9bfb 100644 --- a/transform/array_join.go +++ b/transform/array_join.go @@ -16,6 +16,7 @@ type arrayJoinConfig struct { // Separator is the string that is used to join data. Separator string `json:"separator"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -38,11 +39,15 @@ func (c *arrayJoinConfig) Validate() error { func newArrayJoin(_ context.Context, cfg config.Config) (*arrayJoin, error) { conf := arrayJoinConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: array_join: %v", err) + return nil, fmt.Errorf("transform array_join: %v", err) + } + + if conf.ID == "" { + conf.ID = "array_join" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: array_join: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := arrayJoin{ @@ -92,7 +97,7 @@ func (tf *arrayJoin) Transform(ctx context.Context, msg *message.Message) ([]*me if tf.hasObjectSetKey { if err := msg.SetValue(tf.conf.Object.TargetKey, str); err != nil { - return nil, fmt.Errorf("transform: array_join: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/array_zip.go b/transform/array_zip.go index edee8ff7..778d9e72 100644 --- a/transform/array_zip.go +++ b/transform/array_zip.go @@ -12,6 +12,7 @@ import ( ) type arrayZipConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,11 +35,15 @@ func (c *arrayZipConfig) Validate() error { func newArrayZip(_ context.Context, cfg config.Config) (*arrayZip, error) { conf := arrayZipConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: array_zip: %v", err) + return nil, fmt.Errorf("transform array_zip: %v", err) + } + + if conf.ID == "" { + conf.ID = "array_zip" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: array_zip: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := arrayZip{ @@ -86,7 +91,7 @@ func (tf *arrayZip) Transform(ctx context.Context, msg *message.Message) ([]*mes if tf.hasObjDst { if err := msg.SetValue(tf.conf.Object.TargetKey, b); err != nil { - return nil, fmt.Errorf("transform: array_zip: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich.go b/transform/enrich.go index 2c4e6731..64e19f33 100644 --- a/transform/enrich.go +++ b/transform/enrich.go @@ -15,6 +15,7 @@ import ( const enrichHTTPInterp = `${DATA}` type enrichDNSConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` Request iconfig.Request `json:"request"` } diff --git a/transform/enrich_aws_dynamodb.go b/transform/enrich_aws_dynamodb.go index 1a2c440d..90583e4c 100644 --- a/transform/enrich_aws_dynamodb.go +++ b/transform/enrich_aws_dynamodb.go @@ -40,6 +40,7 @@ type enrichAWSDynamoDBConfig struct { // This is optional and defaults to true. ScanIndexForward bool `json:"scan_index_forward"` + ID string `json:"id"` Object iconfig.Object `json:"object"` AWS iconfig.AWS `json:"aws"` Retry iconfig.Retry `json:"retry"` @@ -71,11 +72,15 @@ func (c *enrichAWSDynamoDBConfig) Validate() error { func newEnrichAWSDynamoDB(_ context.Context, cfg config.Config) (*enrichAWSDynamoDB, error) { conf := enrichAWSDynamoDBConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform enrich_aws_dynamodb: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_aws_dynamodb" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichAWSDynamoDB{ @@ -114,7 +119,7 @@ func (tf *enrichAWSDynamoDB) Transform(ctx context.Context, msg *message.Message } if !json.Valid(tmp.Data()) { - return nil, fmt.Errorf("transform: enrich_aws_dynamodb: %v", errMsgInvalidObject) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errMsgInvalidObject) } pk := tmp.GetValue(tf.conf.PartitionKey) @@ -125,7 +130,7 @@ func (tf *enrichAWSDynamoDB) Transform(ctx context.Context, msg *message.Message sk := tmp.GetValue(tf.conf.SortKey) value, err := tf.dynamodb(ctx, pk.String(), sk.String()) if err != nil { - return nil, fmt.Errorf("transform: enrich_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // No match. @@ -134,7 +139,7 @@ func (tf *enrichAWSDynamoDB) Transform(ctx context.Context, msg *message.Message } if err := msg.SetValue(tf.conf.Object.TargetKey, value); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_aws_lambda.go b/transform/enrich_aws_lambda.go index 8a0450d2..85415be6 100644 --- a/transform/enrich_aws_lambda.go +++ b/transform/enrich_aws_lambda.go @@ -18,6 +18,7 @@ type enrichAWSLambdaConfig struct { // FunctionName is the AWS Lambda function to synchronously invoke. FunctionName string `json:"function_name"` + ID string `json:"id"` Object iconfig.Object `json:"object"` AWS iconfig.AWS `json:"aws"` Retry iconfig.Retry `json:"retry"` @@ -46,11 +47,15 @@ func (c *enrichAWSLambdaConfig) Validate() error { func newEnrichAWSLambda(_ context.Context, cfg config.Config) (*enrichAWSLambda, error) { conf := enrichAWSLambdaConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", err) + return nil, fmt.Errorf("transform enrich_aws_lambda: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_aws_lambda" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichAWSLambda{ @@ -86,21 +91,21 @@ func (tf *enrichAWSLambda) Transform(ctx context.Context, msg *message.Message) } if !json.Valid(value.Bytes()) { - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", errMsgInvalidObject) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errMsgInvalidObject) } resp, err := tf.client.Invoke(ctx, tf.conf.FunctionName, value.Bytes()) if err != nil { - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if resp.FunctionError != nil { resErr := gjson.GetBytes(resp.Payload, "errorMessage").String() - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", resErr) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, resErr) } if err := msg.SetValue(tf.conf.Object.TargetKey, resp.Payload); err != nil { - return nil, fmt.Errorf("transform: enrich_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_dns_domain_lookup.go b/transform/enrich_dns_domain_lookup.go index 6ea0bfc8..0f0f4a9e 100644 --- a/transform/enrich_dns_domain_lookup.go +++ b/transform/enrich_dns_domain_lookup.go @@ -16,16 +16,20 @@ import ( func newEnrichDNSDomainLookup(_ context.Context, cfg config.Config) (*enrichDNSDomainLookup, error) { conf := enrichDNSConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: %v", err) + return nil, fmt.Errorf("transform enrich_dns_domain_lookup: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_dns_domain_lookup" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } dur, err := time.ParseDuration(conf.Request.Timeout) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: duration: %v", err) + return nil, fmt.Errorf("transform %s: duration: %v", conf.ID, err) } tf := enrichDNSDomainLookup{ @@ -59,7 +63,7 @@ func (tf *enrichDNSDomainLookup) Transform(ctx context.Context, msg *message.Mes str := string(msg.Data()) names, err := tf.resolver.LookupHost(resolverCtx, str) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // Return the first name. @@ -75,11 +79,11 @@ func (tf *enrichDNSDomainLookup) Transform(ctx context.Context, msg *message.Mes names, err := tf.resolver.LookupHost(resolverCtx, value.String()) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, names); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_domain_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_dns_ip_lookup.go b/transform/enrich_dns_ip_lookup.go index 01e2b658..e41dec2d 100644 --- a/transform/enrich_dns_ip_lookup.go +++ b/transform/enrich_dns_ip_lookup.go @@ -16,16 +16,20 @@ import ( func newEnrichDNSIPLookup(_ context.Context, cfg config.Config) (*enrichDNSIPLookup, error) { conf := enrichDNSConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: %v", err) + return nil, fmt.Errorf("transform enrich_dns_ip_lookup: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_dns_ip_lookup" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } dur, err := time.ParseDuration(conf.Request.Timeout) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: duration: %v", err) + return nil, fmt.Errorf("transform %s: duration: %v", conf.ID, err) } tf := enrichDNSIPLookup{ @@ -59,7 +63,7 @@ func (tf *enrichDNSIPLookup) Transform(ctx context.Context, msg *message.Message str := string(msg.Data()) addrs, err := tf.resolver.LookupAddr(resolverCtx, str) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // Return the first address. @@ -75,11 +79,11 @@ func (tf *enrichDNSIPLookup) Transform(ctx context.Context, msg *message.Message addrs, err := tf.resolver.LookupAddr(resolverCtx, value.String()) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, addrs); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_ip_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_dns_txt_lookup.go b/transform/enrich_dns_txt_lookup.go index a74e5972..8720e73d 100644 --- a/transform/enrich_dns_txt_lookup.go +++ b/transform/enrich_dns_txt_lookup.go @@ -16,16 +16,20 @@ import ( func newEnrichDNSTxtLookup(_ context.Context, cfg config.Config) (*enrichDNSTxtLookup, error) { conf := enrichDNSConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: %v", err) + return nil, fmt.Errorf("transform enrich_dns_txt_lookup: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_dns_txt_lookup" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } dur, err := time.ParseDuration(conf.Request.Timeout) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: duration: %v", err) + return nil, fmt.Errorf("transform %s: duration: %v", conf.ID, err) } tf := enrichDNSTxtLookup{ @@ -59,7 +63,7 @@ func (tf *enrichDNSTxtLookup) Transform(ctx context.Context, msg *message.Messag str := string(msg.Data()) recs, err := tf.resolver.LookupTXT(resolverCtx, str) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // Return the first record. @@ -75,11 +79,11 @@ func (tf *enrichDNSTxtLookup) Transform(ctx context.Context, msg *message.Messag recs, err := tf.resolver.LookupTXT(resolverCtx, value.String()) if err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, recs); err != nil { - return nil, fmt.Errorf("transform: enrich_dns_txt_lookup: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_http_get.go b/transform/enrich_http_get.go index 271be073..ffc25c81 100644 --- a/transform/enrich_http_get.go +++ b/transform/enrich_http_get.go @@ -29,6 +29,7 @@ type enrichHTTPGetConfig struct { // This is optional and has no default. Headers map[string]string `json:"headers"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -55,11 +56,15 @@ func (c *enrichHTTPGetConfig) Validate() error { func newEnrichHTTPGet(ctx context.Context, cfg config.Config) (*enrichHTTPGet, error) { conf := enrichHTTPGetConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform enrich_http_get: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_http_get" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichHTTPGet{ @@ -71,7 +76,7 @@ func newEnrichHTTPGet(ctx context.Context, cfg config.Config) (*enrichHTTPGet, e // Retrieve secret and interpolate with header value. v, err := secrets.Interpolate(ctx, v) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.headers = append(tf.headers, http.Header{ @@ -124,23 +129,23 @@ func (tf *enrichHTTPGet) Transform(ctx context.Context, msg *message.Message) ([ // Retrieve secret and interpolate with URL url, err := secrets.Interpolate(ctx, url) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // resp.Body is closed by enrichHTTPParseResponse. resp, err := tf.client.Get(ctx, url, tf.headers...) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } parsed, err := enrichHTTPParseResponse(resp) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if tf.conf.Object.TargetKey != "" { if err := msg.SetValue(tf.conf.Object.TargetKey, parsed); err != nil { - return nil, fmt.Errorf("transform: enrich_http_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_http_post.go b/transform/enrich_http_post.go index ac614466..8396aced 100644 --- a/transform/enrich_http_post.go +++ b/transform/enrich_http_post.go @@ -37,6 +37,7 @@ type enrichHTTPPostConfig struct { // This is optional and has no default. Headers map[string]string `json:"headers"` + ID string `json:"id"` Object enrichHTTPPostObjectConfig `json:"object"` } @@ -67,11 +68,15 @@ func (c *enrichHTTPPostConfig) Validate() error { func newEnrichHTTPPost(ctx context.Context, cfg config.Config) (*enrichHTTPPost, error) { conf := enrichHTTPPostConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform enrich_http_post: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_http_post" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichHTTPPost{ @@ -83,7 +88,7 @@ func newEnrichHTTPPost(ctx context.Context, cfg config.Config) (*enrichHTTPPost, // Retrieve secret and interpolate with header value. v, err := secrets.Interpolate(ctx, v) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.headers = append(tf.headers, http.Header{ @@ -136,7 +141,7 @@ func (tf *enrichHTTPPost) Transform(ctx context.Context, msg *message.Message) ( // Retrieve secret and interpolate with URL url, err := secrets.Interpolate(ctx, url) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } bodyValue := msg.GetValue(tf.conf.Object.BodyKey) @@ -149,12 +154,12 @@ func (tf *enrichHTTPPost) Transform(ctx context.Context, msg *message.Message) ( // If ErrorOnFailure is configured, then errors are returned, // but otherwise the message is returned as-is. if err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } parsed, err := enrichHTTPParseResponse(resp) if err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If SetKey exists, then the response body is written into the message, @@ -162,7 +167,7 @@ func (tf *enrichHTTPPost) Transform(ctx context.Context, msg *message.Message) ( // as-is. if tf.conf.Object.TargetKey != "" { if err := msg.SetValue(tf.conf.Object.TargetKey, parsed); err != nil { - return nil, fmt.Errorf("transform: enrich_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } diff --git a/transform/enrich_kv_store_get.go b/transform/enrich_kv_store_get.go index 8052e911..8a6be62d 100644 --- a/transform/enrich_kv_store_get.go +++ b/transform/enrich_kv_store_get.go @@ -26,6 +26,7 @@ type enrichKVStoreGetConfig struct { // This is optional and defaults to false (KV store is not closed). CloseKVStore bool `json:"close_kv_store"` + ID string `json:"id"` Object iconfig.Object `json:"object"` KVStore config.Config `json:"kv_store"` } @@ -53,16 +54,20 @@ func (c *enrichKVStoreGetConfig) Validate() error { func newEnrichKVStoreGet(_ context.Context, cfg config.Config) (*enrichKVStoreGet, error) { conf := enrichKVStoreGetConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform enrich_kv_store_get: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_kv_store_get" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } kvStore, err := kv.Get(conf.KVStore) if err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichKVStoreGet{ @@ -85,7 +90,7 @@ func (tf *enrichKVStoreGet) Transform(ctx context.Context, msg *message.Message) } if err := tf.kvStore.Close(); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil @@ -93,7 +98,7 @@ func (tf *enrichKVStoreGet) Transform(ctx context.Context, msg *message.Message) if !tf.kvStore.IsEnabled() { if err := tf.kvStore.Setup(ctx); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -109,11 +114,11 @@ func (tf *enrichKVStoreGet) Transform(ctx context.Context, msg *message.Message) v, err := tf.kvStore.Get(ctx, key) if err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, v); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_get: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/enrich_kv_store_set.go b/transform/enrich_kv_store_set.go index 8d5cdc39..cd14f44d 100644 --- a/transform/enrich_kv_store_set.go +++ b/transform/enrich_kv_store_set.go @@ -49,6 +49,7 @@ type enrichKVStoreSetConfig struct { // This is optional and defaults to false (KV store is not closed). CloseKVStore bool `json:"close_kv_store"` + ID string `json:"id"` Object enrichKVStoreSetObjectConfig `json:"object"` KVStore config.Config `json:"kv_store"` } @@ -76,16 +77,20 @@ func (c *enrichKVStoreSetConfig) Validate() error { func newEnrichKVStoreSet(_ context.Context, cfg config.Config) (*enrichKVStoreSet, error) { conf := enrichKVStoreSetConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform enrich_kv_store_set: %v", err) + } + + if conf.ID == "" { + conf.ID = "enrich_kv_store_set" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } kvStore, err := kv.Get(conf.KVStore) if err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } if conf.TTLOffset == "" { @@ -94,7 +99,7 @@ func newEnrichKVStoreSet(_ context.Context, cfg config.Config) (*enrichKVStoreSe dur, err := time.ParseDuration(conf.TTLOffset) if err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := enrichKVStoreSet{ @@ -119,7 +124,7 @@ func (tf *enrichKVStoreSet) Transform(ctx context.Context, msg *message.Message) } if err := tf.kvStore.Close(); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil @@ -127,7 +132,7 @@ func (tf *enrichKVStoreSet) Transform(ctx context.Context, msg *message.Message) if !tf.kvStore.IsEnabled() { if err := tf.kvStore.Setup(ctx); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -147,24 +152,24 @@ func (tf *enrichKVStoreSet) Transform(ctx context.Context, msg *message.Message) ttl := truncateTTL(value) + tf.ttl if err := tf.kvStore.SetWithTTL(ctx, key, msg.GetValue(tf.conf.Object.TargetKey).String(), ttl); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else if tf.conf.Object.TTLKey != "" { value := msg.GetValue(tf.conf.Object.TTLKey) ttl := truncateTTL(value) if err := tf.kvStore.SetWithTTL(ctx, key, msg.GetValue(tf.conf.Object.TargetKey).String(), ttl); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else if tf.ttl != 0 { ttl := time.Now().Add(time.Duration(tf.ttl) * time.Second).Unix() if err := tf.kvStore.SetWithTTL(ctx, key, msg.GetValue(tf.conf.Object.TargetKey).String(), ttl); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { if err := tf.kvStore.Set(ctx, key, msg.GetValue(tf.conf.Object.TargetKey).String()); err != nil { - return nil, fmt.Errorf("transform: enrich_kv_store_set: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } diff --git a/transform/format.go b/transform/format.go index f7e1fa2b..01f86ee7 100644 --- a/transform/format.go +++ b/transform/format.go @@ -11,6 +11,7 @@ import ( ) type formatBase64Config struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -30,7 +31,9 @@ func (c *formatBase64Config) Validate() error { return nil } -type formatGzipConfig struct{} +type formatGzipConfig struct { + ID string `json:"id"` +} func (c *formatGzipConfig) Decode(in interface{}) error { return iconfig.Decode(in, c) diff --git a/transform/format_from_base64.go b/transform/format_from_base64.go index 3b278879..cf2fc67e 100644 --- a/transform/format_from_base64.go +++ b/transform/format_from_base64.go @@ -19,11 +19,15 @@ var errFormatFromBase64DecodeBinary = fmt.Errorf("cannot write binary as object" func newFormatFromBase64(_ context.Context, cfg config.Config) (*formatFromBase64, error) { conf := formatBase64Config{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: format_from_base64: %v", err) + return nil, fmt.Errorf("transform format_from_base64: %v", err) + } + + if conf.ID == "" { + conf.ID = "format_from_base64" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: format_from_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := formatFromBase64{ @@ -47,7 +51,7 @@ func (tf *formatFromBase64) Transform(ctx context.Context, msg *message.Message) if !tf.isObject { decoded, err := ibase64.Decode(msg.Data()) if err != nil { - return nil, fmt.Errorf("transform: format_from_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData(decoded) @@ -61,7 +65,7 @@ func (tf *formatFromBase64) Transform(ctx context.Context, msg *message.Message) b64, err := ibase64.Decode(value.Bytes()) if err != nil { - return nil, fmt.Errorf("transform: format_from_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if !utf8.Valid(b64) { @@ -69,7 +73,7 @@ func (tf *formatFromBase64) Transform(ctx context.Context, msg *message.Message) } if err := msg.SetValue(tf.conf.Object.TargetKey, b64); err != nil { - return nil, fmt.Errorf("transform: format_from_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/format_from_gzip.go b/transform/format_from_gzip.go index 593f8908..6ddbb9a6 100644 --- a/transform/format_from_gzip.go +++ b/transform/format_from_gzip.go @@ -12,7 +12,11 @@ import ( func newFormatFromGzip(_ context.Context, cfg config.Config) (*formatFromGzip, error) { conf := formatGzipConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: format_from_gzip: %v", err) + return nil, fmt.Errorf("transform format_from_gzip: %v", err) + } + + if conf.ID == "" { + conf.ID = "format_from_gzip" } tf := formatFromGzip{ @@ -35,7 +39,7 @@ func (tf *formatFromGzip) Transform(ctx context.Context, msg *message.Message) ( gz, err := fmtFromGzip(msg.Data()) if err != nil { - return nil, fmt.Errorf("transform: format_from_gzip: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData(gz) diff --git a/transform/format_from_pretty_print.go b/transform/format_from_pretty_print.go index 522ce668..72bacc1d 100644 --- a/transform/format_from_pretty_print.go +++ b/transform/format_from_pretty_print.go @@ -16,7 +16,9 @@ const ( formatFromPPCloseCurlyBracket = 125 // } ) -type formatFromPrettyPrintConfig struct{} +type formatFromPrettyPrintConfig struct { + ID string `json:"id"` +} func (c *formatFromPrettyPrintConfig) Decode(in interface{}) error { return iconfig.Decode(in, c) @@ -25,7 +27,11 @@ func (c *formatFromPrettyPrintConfig) Decode(in interface{}) error { func newFormatFromPrettyPrint(_ context.Context, cfg config.Config) (*formatFromPrettyPrint, error) { conf := formatFromPrettyPrintConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: format_from_pretty_print: %v", err) + return nil, fmt.Errorf("transform format_from_pretty_print: %v", err) + } + + if conf.ID == "" { + conf.ID = "format_from_pretty_print" } tf := formatFromPrettyPrint{ @@ -61,7 +67,7 @@ func (tf *formatFromPrettyPrint) Transform(ctx context.Context, msg *message.Mes if tf.count == 0 { var buf bytes.Buffer if err := json.Compact(&buf, tf.stack); err != nil { - return nil, fmt.Errorf("transform: format_from_pretty_print: json compact: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } tf.stack = []byte{} @@ -70,7 +76,7 @@ func (tf *formatFromPrettyPrint) Transform(ctx context.Context, msg *message.Mes return []*message.Message{msg}, nil } - return nil, fmt.Errorf("transform: format_from_pretty_print: invalid json") + return nil, fmt.Errorf("transform %s: invalid json", tf.conf.ID) } } diff --git a/transform/format_to_base64.go b/transform/format_to_base64.go index 8ddce7d2..adc634d8 100644 --- a/transform/format_to_base64.go +++ b/transform/format_to_base64.go @@ -13,11 +13,15 @@ import ( func newFormatToBase64(_ context.Context, cfg config.Config) (*formatToBase64, error) { conf := formatBase64Config{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: format_to_base64: %v", err) + return nil, fmt.Errorf("transform format_to_base64: %v", err) + } + + if conf.ID == "" { + conf.ID = "format_to_base64" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: format_to_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := formatToBase64{ @@ -53,7 +57,7 @@ func (tf *formatToBase64) Transform(ctx context.Context, msg *message.Message) ( b64 := ibase64.Encode(value.Bytes()) if err := msg.SetValue(tf.conf.Object.TargetKey, b64); err != nil { - return nil, fmt.Errorf("transform: format_to_base64: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/format_to_gzip.go b/transform/format_to_gzip.go index e7e8502d..543798c3 100644 --- a/transform/format_to_gzip.go +++ b/transform/format_to_gzip.go @@ -12,7 +12,11 @@ import ( func newFormatToGzip(_ context.Context, cfg config.Config) (*formatToGzip, error) { conf := formatGzipConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: format_to_gzip: %v", err) + return nil, fmt.Errorf("transform format_to_gzip: %v", err) + } + + if conf.ID == "" { + conf.ID = "format_to_gzip" } tf := formatToGzip{ @@ -35,7 +39,7 @@ func (tf *formatToGzip) Transform(ctx context.Context, msg *message.Message) ([] gz, err := fmtToGzip(msg.Data()) if err != nil { - return nil, fmt.Errorf("transform: format_to_gzip: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData(gz) diff --git a/transform/hash.go b/transform/hash.go index bb6e6669..5526e291 100644 --- a/transform/hash.go +++ b/transform/hash.go @@ -8,6 +8,7 @@ import ( ) type hashConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } diff --git a/transform/hash_md5.go b/transform/hash_md5.go index 11f6dc91..3083dbb3 100644 --- a/transform/hash_md5.go +++ b/transform/hash_md5.go @@ -13,11 +13,15 @@ import ( func newHashMD5(_ context.Context, cfg config.Config) (*hashMD5, error) { conf := hashConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: hash_md5: %v", err) + return nil, fmt.Errorf("transform hash_md5: %v", err) + } + + if conf.ID == "" { + conf.ID = "hash_md5" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: hash_md5: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := hashMD5{ diff --git a/transform/hash_sha256.go b/transform/hash_sha256.go index 1367344e..f70c860f 100644 --- a/transform/hash_sha256.go +++ b/transform/hash_sha256.go @@ -13,11 +13,15 @@ import ( func newHashSHA256(_ context.Context, cfg config.Config) (*hashSHA256, error) { conf := hashConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: hash_sha256: %v", err) + return nil, fmt.Errorf("transform hash_sha256: %v", err) + } + + if conf.ID == "" { + conf.ID = "hash_sha256" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: hash_sha256: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := hashSHA256{ diff --git a/transform/meta_err.go b/transform/meta_err.go index 6cc7d499..6df0fb55 100644 --- a/transform/meta_err.go +++ b/transform/meta_err.go @@ -20,6 +20,8 @@ type metaErrConfig struct { // // This is optional and defaults to an empty list (all errors are caught). ErrorMessages []string `json:"error_messages"` + + ID string `json:"id"` } func (c *metaErrConfig) Decode(in interface{}) error { @@ -37,23 +39,27 @@ func (c *metaErrConfig) Validate() error { func newMetaErr(ctx context.Context, cfg config.Config) (*metaErr, error) { conf := metaErrConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_err: %v", err) + return nil, fmt.Errorf("transform meta_err: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_err" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: meta_err: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf, err := New(ctx, conf.Transform) if err != nil { - return nil, fmt.Errorf("transform: meta_err: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } errMsgs := make([]*regexp.Regexp, len(conf.ErrorMessages)) for i, eMsg := range conf.ErrorMessages { r, err := regexp.Compile(eMsg) if err != nil { - return nil, fmt.Errorf("transform: meta_err: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } errMsgs[i] = r @@ -88,7 +94,7 @@ func (tf *metaErr) Transform(ctx context.Context, msg *message.Message) ([]*mess } } - return nil, fmt.Errorf("transform: meta_err: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil diff --git a/transform/meta_for_each.go b/transform/meta_for_each.go index 2d2c5961..0b4fbac7 100644 --- a/transform/meta_for_each.go +++ b/transform/meta_for_each.go @@ -15,6 +15,7 @@ type metaForEachConfig struct { // Transform that is applied to each item in the array. Transform config.Config `json:"transform"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -41,11 +42,15 @@ func (c *metaForEachConfig) Validate() error { func newMetaForEach(ctx context.Context, cfg config.Config) (*metaForEach, error) { conf := metaForEachConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform meta_for_each: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_for_each" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := metaForEach{ @@ -63,7 +68,7 @@ func newMetaForEach(ctx context.Context, cfg config.Config) (*metaForEach, error tfer, err := New(ctx, tf.tfCfg) if err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tf = tfer @@ -81,7 +86,7 @@ func (tf *metaForEach) Transform(ctx context.Context, msg *message.Message) ([]* if msg.IsControl() { msgs, err := tf.tf.Transform(ctx, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil @@ -101,7 +106,7 @@ func (tf *metaForEach) Transform(ctx context.Context, msg *message.Message) ([]* tmpMsg := message.New().SetData(res.Bytes()) tfMsgs, err := tf.tf.Transform(ctx, tmpMsg) if err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } for _, m := range tfMsgs { @@ -111,7 +116,7 @@ func (tf *metaForEach) Transform(ctx context.Context, msg *message.Message) ([]* } if err := msg.SetValue(tf.conf.Object.TargetKey, arr); err != nil { - return nil, fmt.Errorf("transform: meta_for_each: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/meta_kv_store_lock.go b/transform/meta_kv_store_lock.go index 12cec07d..47157dab 100644 --- a/transform/meta_kv_store_lock.go +++ b/transform/meta_kv_store_lock.go @@ -43,6 +43,7 @@ type metaKVStoreLockConfig struct { // This is optional and defaults to using no TTL when setting values into the store. TTLOffset string `json:"ttl_offset"` + ID string `json:"id"` Transform config.Config `json:"transform"` Object metaVStoreLockObjectConfig `json:"object"` KVStore config.Config `json:"kv_store"` @@ -67,25 +68,29 @@ func (c *metaKVStoreLockConfig) Validate() error { func newMetaKVStoreLock(ctx context.Context, cfg config.Config) (*metaKVStoreLock, error) { conf := metaKVStoreLockConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform meta_kv_store_lock: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_kv_store_lock" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tff, err := New(ctx, conf.Transform) if err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } locker, err := kv.GetLocker(conf.KVStore) if err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } if err := locker.Setup(ctx); err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } if conf.TTLOffset == "" { @@ -94,7 +99,7 @@ func newMetaKVStoreLock(ctx context.Context, cfg config.Config) (*metaKVStoreLoc dur, err := time.ParseDuration(conf.TTLOffset) if err != nil { - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := metaKVStoreLock{ @@ -133,7 +138,7 @@ func (tf *metaKVStoreLock) Transform(ctx context.Context, msg *message.Message) _ = tf.locker.Unlock(ctx, key) } - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } tf.keys = tf.keys[:0] @@ -177,7 +182,7 @@ func (tf *metaKVStoreLock) Transform(ctx context.Context, msg *message.Message) _ = tf.locker.Unlock(ctx, key) } - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } tf.keys = append(tf.keys, lockKey) @@ -187,7 +192,7 @@ func (tf *metaKVStoreLock) Transform(ctx context.Context, msg *message.Message) _ = tf.locker.Unlock(ctx, key) } - return nil, fmt.Errorf("transform: meta_kv_store_lock: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil diff --git a/transform/meta_metric_duration.go b/transform/meta_metric_duration.go index 6673cf8d..5bba6274 100644 --- a/transform/meta_metric_duration.go +++ b/transform/meta_metric_duration.go @@ -13,6 +13,7 @@ import ( ) type metaMetricDurationConfig struct { + ID string `json:"id"` Metric iconfig.Metric `json:"metric"` Transform config.Config `json:"transform"` } @@ -25,12 +26,16 @@ func newMetaMetricsDuration(ctx context.Context, cfg config.Config) (*metaMetric // conf gets validated when calling metrics.New. conf := metaMetricDurationConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_metric_duration: %v", err) + return nil, fmt.Errorf("transform meta_metric_duration: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_metric_duration" } m, err := metrics.New(ctx, conf.Metric.Destination) if err != nil { - return nil, fmt.Errorf("transform: meta_metric_duration: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := metaMetricDuration{ @@ -50,7 +55,7 @@ func newMetaMetricsDuration(ctx context.Context, cfg config.Config) (*metaMetric tfer, err := New(ctx, tfCfg) if err != nil { - return nil, fmt.Errorf("transform: meta_metric_duration: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tf = tfer @@ -74,12 +79,12 @@ func (tf *metaMetricDuration) Transform(ctx context.Context, msg *message.Messag Value: tf.duration, Attributes: tf.conf.Metric.Attributes, }); err != nil { - return nil, fmt.Errorf("transform: meta_metric_duration: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msgs, err := tf.tf.Transform(ctx, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_metric_duration: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil diff --git a/transform/meta_pipeline.go b/transform/meta_pipeline.go index 750f82cf..f64b3efa 100644 --- a/transform/meta_pipeline.go +++ b/transform/meta_pipeline.go @@ -21,6 +21,7 @@ type metaPipelineConfig struct { // Transforms that is applied in series to the data. Transforms []config.Config `json:"transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -47,11 +48,15 @@ func (c *metaPipelineConfig) Validate() error { func newMetaPipeline(ctx context.Context, cfg config.Config) (*metaPipeline, error) { conf := metaPipelineConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform meta_pipeline: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_pipeline" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := metaPipeline{ @@ -63,7 +68,7 @@ func newMetaPipeline(ctx context.Context, cfg config.Config) (*metaPipeline, err for _, c := range conf.Transforms { t, err := New(ctx, c) if err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: transform %+v: %v", c, err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tform = append(tform, t) @@ -84,7 +89,7 @@ func (tf *metaPipeline) Transform(ctx context.Context, msg *message.Message) ([] if msg.IsControl() { msgs, err := Apply(ctx, tf.tf, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil @@ -93,7 +98,7 @@ func (tf *metaPipeline) Transform(ctx context.Context, msg *message.Message) ([] if !tf.isObject { msgs, err := Apply(ctx, tf.tf, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil @@ -105,18 +110,18 @@ func (tf *metaPipeline) Transform(ctx context.Context, msg *message.Message) ([] } if value.IsArray() { - return nil, fmt.Errorf("transform: meta_pipeline: key %s: %v", tf.conf.Object.SourceKey, errMetaPipelineArrayInput) + return nil, fmt.Errorf("transform %s: key %s: %v", tf.conf.ID, tf.conf.Object.SourceKey, errMetaPipelineArrayInput) } res, err := Apply(ctx, tf.tf, message.New().SetData(value.Bytes())) if err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } var output []*message.Message for _, msg := range res { if err := msg.SetValue(tf.conf.Object.TargetKey, msg.Data()); err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } output = append(output, msg) diff --git a/transform/meta_switch.go b/transform/meta_switch.go index 317497f9..b2d8b33c 100644 --- a/transform/meta_switch.go +++ b/transform/meta_switch.go @@ -20,6 +20,8 @@ type metaSwitchConfig struct { Condition condition.Config `json:"condition"` Transform config.Config `json:"transform"` } `json:"cases"` + + ID string `json:"id"` } func (c *metaSwitchConfig) Decode(in interface{}) error { @@ -37,11 +39,15 @@ func (c *metaSwitchConfig) Validate() error { func newMetaSwitch(ctx context.Context, cfg config.Config) (*metaSwitch, error) { conf := metaSwitchConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform meta_switch: %v", err) + } + + if conf.ID == "" { + conf.ID = "meta_switch" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } var conditional []struct { @@ -51,12 +57,12 @@ func newMetaSwitch(ctx context.Context, cfg config.Config) (*metaSwitch, error) for _, s := range conf.Cases { op, err := condition.New(ctx, s.Condition) if err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf, err := New(ctx, s.Transform) if err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform meta_switch: %v", err) } conditional = append(conditional, struct { @@ -91,7 +97,7 @@ func (tf *metaSwitch) Transform(ctx context.Context, msg *message.Message) ([]*m for _, c := range tf.conditional { res, err := c.tf.Transform(ctx, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_pipeline: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } messages = append(messages, res...) @@ -115,7 +121,7 @@ func (tf *metaSwitch) Transform(ctx context.Context, msg *message.Message) ([]*m for _, c := range tf.conditional { ok, err := c.op.Operate(ctx, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if !ok { @@ -124,7 +130,7 @@ func (tf *metaSwitch) Transform(ctx context.Context, msg *message.Message) ([]*m msgs, err := c.tf.Transform(ctx, msg) if err != nil { - return nil, fmt.Errorf("transform: meta_switch: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return msgs, nil diff --git a/transform/network.go b/transform/network.go index eec02bb9..82ab4e52 100644 --- a/transform/network.go +++ b/transform/network.go @@ -8,6 +8,7 @@ import ( ) type networkDomainConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } diff --git a/transform/network_domain_registered_domain.go b/transform/network_domain_registered_domain.go index efd8d26f..da59e94d 100644 --- a/transform/network_domain_registered_domain.go +++ b/transform/network_domain_registered_domain.go @@ -14,11 +14,15 @@ import ( func newNetworkDomainRegisteredDomain(_ context.Context, cfg config.Config) (*networkDomainRegisteredDomain, error) { conf := networkDomainConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: network_domain_registered_domain: %v", err) + return nil, fmt.Errorf("transform network_domain_registered_domain: %v", err) + } + + if conf.ID == "" { + conf.ID = "network_domain_registered_domain" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: network_domain_registered_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := networkDomainRegisteredDomain{ @@ -43,7 +47,7 @@ func (tf *networkDomainRegisteredDomain) Transform(ctx context.Context, msg *mes str := string(msg.Data()) domain, err := publicsuffix.EffectiveTLDPlusOne(str) if err != nil { - return nil, fmt.Errorf("transform: network_domain_registered_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData([]byte(domain)) @@ -57,11 +61,11 @@ func (tf *networkDomainRegisteredDomain) Transform(ctx context.Context, msg *mes domain, err := publicsuffix.EffectiveTLDPlusOne(value.String()) if err != nil { - return nil, fmt.Errorf("transform: network_domain_registered_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, domain); err != nil { - return nil, fmt.Errorf("transform: network_domain_registered_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/network_domain_subdomain.go b/transform/network_domain_subdomain.go index 1bcb0abb..cba23bcb 100644 --- a/transform/network_domain_subdomain.go +++ b/transform/network_domain_subdomain.go @@ -19,11 +19,15 @@ var errFmtSubdomainNoSubdomain = fmt.Errorf("no subdomain") func newNetworkDomainSubdomain(_ context.Context, cfg config.Config) (*networkDomainSubdomain, error) { conf := networkDomainConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: network_domain_subdomain: %v", err) + return nil, fmt.Errorf("transform network_domain_subdomain: %v", err) + } + + if conf.ID == "" { + conf.ID = "network_domain_subdomain" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: network_domain_subdomain: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := networkDomainSubdomain{ @@ -48,7 +52,7 @@ func (tf *networkDomainSubdomain) Transform(ctx context.Context, msg *message.Me str := string(msg.Data()) domain, err := fmtParseSubdomain(str) if err != nil { - return nil, fmt.Errorf("transform: network_domain_subdomain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData([]byte(domain)) @@ -62,11 +66,11 @@ func (tf *networkDomainSubdomain) Transform(ctx context.Context, msg *message.Me domain, err := fmtParseSubdomain(value.String()) if err != nil { - return nil, fmt.Errorf("transform: network_domain_subdomain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, domain); err != nil { - return nil, fmt.Errorf("transform: network_domain_subdomain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/network_domain_top_level_domain.go b/transform/network_domain_top_level_domain.go index 48b59ece..79acca37 100644 --- a/transform/network_domain_top_level_domain.go +++ b/transform/network_domain_top_level_domain.go @@ -14,11 +14,15 @@ import ( func newNetworkDomainTopLevelDomain(_ context.Context, cfg config.Config) (*networkDomainTopLevelDomain, error) { conf := networkDomainConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: network_domain_top_level_domain: %v", err) + return nil, fmt.Errorf("transform network_domain_top_level_domain: %v", err) + } + + if conf.ID == "" { + conf.ID = "network_domain_top_level_domain" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: network_domain_top_level_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := networkDomainTopLevelDomain{ @@ -55,7 +59,7 @@ func (tf *networkDomainTopLevelDomain) Transform(ctx context.Context, msg *messa domain, _ := publicsuffix.PublicSuffix(value.String()) if err := msg.SetValue(tf.conf.Object.TargetKey, domain); err != nil { - return nil, fmt.Errorf("transform: network_domain_top_level_domain: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/number.go b/transform/number.go index 740c5dbc..51193fec 100644 --- a/transform/number.go +++ b/transform/number.go @@ -9,6 +9,7 @@ import ( ) type numberMathConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } diff --git a/transform/number_math_addition.go b/transform/number_math_addition.go index 677d4565..02f673ea 100644 --- a/transform/number_math_addition.go +++ b/transform/number_math_addition.go @@ -14,11 +14,15 @@ import ( func newNumberMathAddition(_ context.Context, cfg config.Config) (*numberMathAddition, error) { conf := numberMathConfig{} if err := iconfig.Decode(cfg.Settings, &conf); err != nil { - return nil, fmt.Errorf("transform: number_math_addition: %v", err) + return nil, fmt.Errorf("transform number_math_addition: %v", err) + } + + if conf.ID == "" { + conf.ID = "number_math_addition" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: number_math_addition: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := numberMathAddition{ @@ -73,11 +77,11 @@ func (tf *numberMathAddition) Transform(ctx context.Context, msg *message.Messag f, err := strconv.ParseFloat(strFloat64, 64) if err != nil { - return nil, fmt.Errorf("transform: number_math_addition: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, f); err != nil { - return nil, fmt.Errorf("transform: number_math_addition: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/number_math_division.go b/transform/number_math_division.go index a6ff99ad..60e2735c 100644 --- a/transform/number_math_division.go +++ b/transform/number_math_division.go @@ -14,11 +14,15 @@ import ( func newNumberMathDivision(_ context.Context, cfg config.Config) (*numberMathDivision, error) { conf := numberMathConfig{} if err := iconfig.Decode(cfg.Settings, &conf); err != nil { - return nil, fmt.Errorf("transform: number_math_division: %v", err) + return nil, fmt.Errorf("transform number_math_division: %v", err) + } + + if conf.ID == "" { + conf.ID = "number_math_division" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: number_math_division: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := numberMathDivision{ @@ -73,11 +77,11 @@ func (tf *numberMathDivision) Transform(ctx context.Context, msg *message.Messag f, err := strconv.ParseFloat(strFloat64, 64) if err != nil { - return nil, fmt.Errorf("transform: number_math_division: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, f); err != nil { - return nil, fmt.Errorf("transform: number_math_division: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/number_math_multiplication.go b/transform/number_math_multiplication.go index 6549c11f..56d27573 100644 --- a/transform/number_math_multiplication.go +++ b/transform/number_math_multiplication.go @@ -14,11 +14,15 @@ import ( func newNumberMathMultiplication(_ context.Context, cfg config.Config) (*numberMathMultiplication, error) { conf := numberMathConfig{} if err := iconfig.Decode(cfg.Settings, &conf); err != nil { - return nil, fmt.Errorf("transform: number_math_multiplication: %v", err) + return nil, fmt.Errorf("transform number_math_multiplication: %v", err) + } + + if conf.ID == "" { + conf.ID = "number_math_multiplication" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: number_math_multiplication: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := numberMathMultiplication{ @@ -73,11 +77,11 @@ func (tf *numberMathMultiplication) Transform(ctx context.Context, msg *message. f, err := strconv.ParseFloat(strFloat64, 64) if err != nil { - return nil, fmt.Errorf("transform: number_math_multiplication: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, f); err != nil { - return nil, fmt.Errorf("transform: number_math_multiplication: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/number_math_subtraction.go b/transform/number_math_subtraction.go index 79f3e945..cca26e92 100644 --- a/transform/number_math_subtraction.go +++ b/transform/number_math_subtraction.go @@ -14,11 +14,15 @@ import ( func newNumberMathSubtraction(_ context.Context, cfg config.Config) (*numberMathSubtraction, error) { conf := numberMathConfig{} if err := iconfig.Decode(cfg.Settings, &conf); err != nil { - return nil, fmt.Errorf("transform: number_math_subtraction: %v", err) + return nil, fmt.Errorf("transform number_math_subtraction: %v", err) + } + + if conf.ID == "" { + conf.ID = "number_math_subtraction" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: number_math_subtraction: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := numberMathSubtraction{ @@ -73,11 +77,11 @@ func (tf *numberMathSubtraction) Transform(ctx context.Context, msg *message.Mes f, err := strconv.ParseFloat(strFloat64, 64) if err != nil { - return nil, fmt.Errorf("transform: number_math_subtraction: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if err := msg.SetValue(tf.conf.Object.TargetKey, f); err != nil { - return nil, fmt.Errorf("transform: number_math_subtraction: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_copy.go b/transform/object_copy.go index 56bdd2f4..edfad8c4 100644 --- a/transform/object_copy.go +++ b/transform/object_copy.go @@ -11,6 +11,7 @@ import ( ) type objectCopyConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -21,7 +22,11 @@ func (c *objectCopyConfig) Decode(in interface{}) error { func newObjectCopy(_ context.Context, cfg config.Config) (*objectCopy, error) { conf := objectCopyConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_copy: %v", err) + return nil, fmt.Errorf("transform object_copy: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_copy" } tf := objectCopy{ @@ -61,7 +66,7 @@ func (tf *objectCopy) Transform(ctx context.Context, msg *message.Message) ([]*m outMsg := message.New().SetMetadata(msg.Metadata()) if err := outMsg.SetValue(tf.conf.Object.TargetKey, msg.Data()); err != nil { - return nil, fmt.Errorf("transform: object_copy: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{outMsg}, nil @@ -73,7 +78,7 @@ func (tf *objectCopy) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := msg.SetValue(tf.conf.Object.TargetKey, value); err != nil { - return nil, fmt.Errorf("transform: object_copy: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_delete.go b/transform/object_delete.go index 20ac6fde..12bb042a 100644 --- a/transform/object_delete.go +++ b/transform/object_delete.go @@ -12,6 +12,7 @@ import ( ) type objectDeleteConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -30,11 +31,15 @@ func (c *objectDeleteConfig) Validate() error { func newObjectDelete(_ context.Context, cfg config.Config) (*objectDelete, error) { conf := objectDeleteConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_delete: %v", err) + return nil, fmt.Errorf("transform object_delete: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_delete" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: object_delete: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } proc := objectDelete{ @@ -54,7 +59,7 @@ func (tf *objectDelete) Transform(_ context.Context, msg *message.Message) ([]*m } if err := msg.DeleteValue(tf.conf.Object.SourceKey); err != nil { - return nil, fmt.Errorf("transform: object_delete: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_insert.go b/transform/object_insert.go index 877eab55..21a78303 100644 --- a/transform/object_insert.go +++ b/transform/object_insert.go @@ -15,6 +15,7 @@ type objectInsertConfig struct { // Value inserted into the object. Value interface{} `json:"value"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -37,11 +38,15 @@ func (c *objectInsertConfig) Validate() error { func newObjectInsert(_ context.Context, cfg config.Config) (*objectInsert, error) { conf := objectInsertConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_insert: %v", err) + return nil, fmt.Errorf("transform object_insert: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_insert" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: object_insert: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := objectInsert{ @@ -61,7 +66,7 @@ func (tf *objectInsert) Transform(ctx context.Context, msg *message.Message) ([] } if err := msg.SetValue(tf.conf.Object.TargetKey, tf.conf.Value); err != nil { - return nil, fmt.Errorf("transform: object_insert: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_jq.go b/transform/object_jq.go index 59131abf..70c68727 100644 --- a/transform/object_jq.go +++ b/transform/object_jq.go @@ -18,6 +18,8 @@ var errObjectJQNoOutputGenerated = fmt.Errorf("no output generated") type objectJQConfig struct { // Filter is the jq filter applied to data. Filter string `json:"filter"` + + ID string `json:"id"` } func (c *objectJQConfig) Decode(in interface{}) error { @@ -35,11 +37,15 @@ func (c *objectJQConfig) Validate() error { func newObjectJQ(_ context.Context, cfg config.Config) (*objectJQ, error) { conf := objectJQConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_jq: %v", err) + return nil, fmt.Errorf("transform object_jq: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_jq" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: object_jq: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } q, err := gojq.Parse(conf.Filter) @@ -68,7 +74,7 @@ func (tf *objectJQ) Transform(ctx context.Context, msg *message.Message) ([]*mes var i interface{} if err := json.Unmarshal(msg.Data(), &i); err != nil { - return nil, fmt.Errorf("transform: object_jq: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } var arr []interface{} @@ -80,7 +86,7 @@ func (tf *objectJQ) Transform(ctx context.Context, msg *message.Message) ([]*mes break } if err, ok := v.(error); ok { - return nil, fmt.Errorf("transform: object_jq: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } arr = append(arr, v) @@ -90,7 +96,7 @@ func (tf *objectJQ) Transform(ctx context.Context, msg *message.Message) ([]*mes var b []byte switch len(arr) { case 0: - return nil, fmt.Errorf("transform: object_jq: %v", errObjectJQNoOutputGenerated) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errObjectJQNoOutputGenerated) case 1: b, err = json.Marshal(arr[0]) default: @@ -98,7 +104,7 @@ func (tf *objectJQ) Transform(ctx context.Context, msg *message.Message) ([]*mes } if err != nil { - return nil, fmt.Errorf("transform: object_jq: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } msg.SetData(b) diff --git a/transform/object_to_boolean.go b/transform/object_to_boolean.go index d8fdd531..e6c84806 100644 --- a/transform/object_to_boolean.go +++ b/transform/object_to_boolean.go @@ -12,6 +12,7 @@ import ( ) type objectToBooleanConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,11 +35,15 @@ func (c *objectToBooleanConfig) Validate() error { func newObjectToBoolean(_ context.Context, cfg config.Config) (*objectToBoolean, error) { conf := objectToBooleanConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_to_boolean: %v", err) + return nil, fmt.Errorf("transform object_to_boolean: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_to_boolean" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: object_to_boolean: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := objectToBoolean{ @@ -68,7 +73,7 @@ func (tf *objectToBoolean) Transform(ctx context.Context, msg *message.Message) } if err := msg.SetValue(tf.conf.Object.TargetKey, value.Bool()); err != nil { - return nil, fmt.Errorf("transform: object_to_boolean: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_to_float.go b/transform/object_to_float.go index 3d1556d0..24d1ac5e 100644 --- a/transform/object_to_float.go +++ b/transform/object_to_float.go @@ -12,6 +12,7 @@ import ( ) type objectToFloatConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,7 +35,11 @@ func (c *objectToFloatConfig) Validate() error { func newObjectToFloat(_ context.Context, cfg config.Config) (*objectToFloat, error) { conf := objectToFloatConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_to_float: %v", err) + return nil, fmt.Errorf("transform object_to_float: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_to_float" } tf := objectToFloat{ @@ -59,7 +64,7 @@ func (tf *objectToFloat) Transform(ctx context.Context, msg *message.Message) ([ } if err := msg.SetValue(tf.conf.Object.TargetKey, value.Float()); err != nil { - return nil, fmt.Errorf("transform: object_to_float: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_to_integer.go b/transform/object_to_integer.go index 7b0114e4..4cdd4fc7 100644 --- a/transform/object_to_integer.go +++ b/transform/object_to_integer.go @@ -12,6 +12,7 @@ import ( ) type objectToIntegerConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,7 +35,11 @@ func (c *objectToIntegerConfig) Validate() error { func newObjectToInteger(_ context.Context, cfg config.Config) (*objectToInteger, error) { conf := objectToIntegerConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_to_integer: %v", err) + return nil, fmt.Errorf("transform object_to_integer: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_to_integer" } tf := objectToInteger{ @@ -59,7 +64,7 @@ func (tf *objectToInteger) Transform(ctx context.Context, msg *message.Message) } if err := msg.SetValue(tf.conf.Object.TargetKey, value.Int()); err != nil { - return nil, fmt.Errorf("transform: object_to_integer: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_to_string.go b/transform/object_to_string.go index a4cc175e..6bf886dd 100644 --- a/transform/object_to_string.go +++ b/transform/object_to_string.go @@ -12,6 +12,7 @@ import ( ) type objectToStringConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,7 +35,11 @@ func (c *objectToStringConfig) Validate() error { func newObjectToString(_ context.Context, cfg config.Config) (*objectToString, error) { conf := objectToStringConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_to_string: %v", err) + return nil, fmt.Errorf("transform object_to_string: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_to_string" } tf := objectToString{ @@ -59,7 +64,7 @@ func (tf *objectToString) Transform(ctx context.Context, msg *message.Message) ( } if err := msg.SetValue(tf.conf.Object.TargetKey, value.String()); err != nil { - return nil, fmt.Errorf("transform: object_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/object_to_unsigned_integer.go b/transform/object_to_unsigned_integer.go index 690215b5..90f59889 100644 --- a/transform/object_to_unsigned_integer.go +++ b/transform/object_to_unsigned_integer.go @@ -12,6 +12,7 @@ import ( ) type objectToUnsignedIntegerConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -34,7 +35,11 @@ func (c *objectToUnsignedIntegerConfig) Validate() error { func newObjectToUnsignedInteger(_ context.Context, cfg config.Config) (*objectToUnsignedInteger, error) { conf := objectToUnsignedIntegerConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: object_to_unsigned_integer: %v", err) + return nil, fmt.Errorf("transform object_to_unsigned_integer: %v", err) + } + + if conf.ID == "" { + conf.ID = "object_to_unsigned_integer" } tf := objectToUnsignedInteger{ @@ -59,7 +64,7 @@ func (tf *objectToUnsignedInteger) Transform(ctx context.Context, msg *message.M } if err := msg.SetValue(tf.conf.Object.TargetKey, value.Uint()); err != nil { - return nil, fmt.Errorf("transform: object_to_unsigned_integer: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/send_aws_dynamodb.go b/transform/send_aws_dynamodb.go index 47450dbd..b29c1098 100644 --- a/transform/send_aws_dynamodb.go +++ b/transform/send_aws_dynamodb.go @@ -37,6 +37,7 @@ type sendAWSDynamoDBConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -58,11 +59,15 @@ func (c *sendAWSDynamoDBConfig) Validate() error { func newSendAWSDynamoDB(_ context.Context, cfg config.Config) (*sendAWSDynamoDB, error) { conf := sendAWSDynamoDBConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform send_aws_dynamodb: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_dynamodb" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSDynamoDB{ @@ -92,7 +97,7 @@ func newSendAWSDynamoDB(_ context.Context, cfg config.Config) (*sendAWSDynamoDB, for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -124,7 +129,7 @@ func (tf *sendAWSDynamoDB) Transform(ctx context.Context, msg *message.Message) } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -133,11 +138,11 @@ func (tf *sendAWSDynamoDB) Transform(ctx context.Context, msg *message.Message) } if !json.Valid(msg.Data()) { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", errSendAWSDynamoDBNonObject) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendAWSDynamoDBNonObject) } if len(msg.Data()) > sendAWSDynamoDBItemSizeLimit { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", errSendAWSDynamoDBItemSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendAWSDynamoDBItemSizeLimit) } // If this value does not exist, then all data is batched together. @@ -147,13 +152,13 @@ func (tf *sendAWSDynamoDB) Transform(ctx context.Context, msg *message.Message) } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_dynamodb: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/send_aws_kinesis_data_firehose.go b/transform/send_aws_kinesis_data_firehose.go index fb275b73..3fc6faf1 100644 --- a/transform/send_aws_kinesis_data_firehose.go +++ b/transform/send_aws_kinesis_data_firehose.go @@ -30,6 +30,7 @@ type sendAWSKinesisDataFirehoseConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -51,11 +52,15 @@ func (c *sendAWSKinesisDataFirehoseConfig) Validate() error { func newSendAWSKinesisDataFirehose(_ context.Context, cfg config.Config) (*sendAWSKinesisDataFirehose, error) { conf := sendAWSKinesisDataFirehoseConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform send_aws_kinesis_data_firehose: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_kinesis_data_firehose" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSKinesisDataFirehose{ @@ -86,7 +91,7 @@ func newSendAWSKinesisDataFirehose(_ context.Context, cfg config.Config) (*sendA for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -118,7 +123,7 @@ func (tf *sendAWSKinesisDataFirehose) Transform(ctx context.Context, msg *messag } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -127,7 +132,7 @@ func (tf *sendAWSKinesisDataFirehose) Transform(ctx context.Context, msg *messag } if len(msg.Data()) > sendAWSKinesisDataFirehoseMessageSizeLimit { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", errSendAWSKinesisDataFirehoseRecordSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendAWSKinesisDataFirehoseRecordSizeLimit) } // If this value does not exist, then all data is batched together. @@ -137,13 +142,13 @@ func (tf *sendAWSKinesisDataFirehose) Transform(ctx context.Context, msg *messag } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_firehose: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/send_aws_kinesis_data_stream.go b/transform/send_aws_kinesis_data_stream.go index e07e617c..ba525f30 100644 --- a/transform/send_aws_kinesis_data_stream.go +++ b/transform/send_aws_kinesis_data_stream.go @@ -36,6 +36,7 @@ type sendAWSKinesisDataStreamConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -57,11 +58,15 @@ func (c *sendAWSKinesisDataStreamConfig) Validate() error { func newSendAWSKinesisDataStream(_ context.Context, cfg config.Config) (*sendAWSKinesisDataStream, error) { conf := sendAWSKinesisDataStreamConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform send_aws_kinesis_data_stream: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_kinesis_data_stream" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSKinesisDataStream{ @@ -75,7 +80,7 @@ func newSendAWSKinesisDataStream(_ context.Context, cfg config.Config) (*sendAWS Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -84,7 +89,7 @@ func newSendAWSKinesisDataStream(_ context.Context, cfg config.Config) (*sendAWS for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -124,7 +129,7 @@ func (tf *sendAWSKinesisDataStream) Transform(ctx context.Context, msg *message. } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -133,7 +138,7 @@ func (tf *sendAWSKinesisDataStream) Transform(ctx context.Context, msg *message. } if len(msg.Data()) > sendAWSKinesisDataStreamMessageSizeLimit { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", errSendAWSKinesisDataStreamMessageSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendAWSKinesisDataStreamMessageSizeLimit) } // If this value does not exist, then all data is batched together. @@ -143,13 +148,13 @@ func (tf *sendAWSKinesisDataStream) Transform(ctx context.Context, msg *message. } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_kinesis_data_stream: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/send_aws_lambda.go b/transform/send_aws_lambda.go index dd5f7868..f8d174a3 100644 --- a/transform/send_aws_lambda.go +++ b/transform/send_aws_lambda.go @@ -30,6 +30,7 @@ type sendAWSLambdaConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -51,11 +52,15 @@ func (c *sendAWSLambdaConfig) Validate() error { func newSendAWSLambda(_ context.Context, cfg config.Config) (*sendAWSLambda, error) { conf := sendAWSLambdaConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform send_aws_lambda: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_lambda" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSLambda{ @@ -69,7 +74,7 @@ func newSendAWSLambda(_ context.Context, cfg config.Config) (*sendAWSLambda, err Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -78,7 +83,7 @@ func newSendAWSLambda(_ context.Context, cfg config.Config) (*sendAWSLambda, err for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -119,7 +124,7 @@ func (tf *sendAWSLambda) Transform(ctx context.Context, msg *message.Message) ([ } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -128,7 +133,7 @@ func (tf *sendAWSLambda) Transform(ctx context.Context, msg *message.Message) ([ } if len(msg.Data()) > sendLambdaPayloadSizeLimit { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", errSendLambdaPayloadSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendLambdaPayloadSizeLimit) } // If this value does not exist, then all data is batched together. @@ -138,13 +143,13 @@ func (tf *sendAWSLambda) Transform(ctx context.Context, msg *message.Message) ([ } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_lambda: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/send_aws_s3.go b/transform/send_aws_s3.go index 91bf378b..0bf666f6 100644 --- a/transform/send_aws_s3.go +++ b/transform/send_aws_s3.go @@ -29,6 +29,7 @@ type sendAWSS3Config struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -50,11 +51,15 @@ func (c *sendAWSS3Config) Validate() error { func newSendAWSS3(_ context.Context, cfg config.Config) (*sendAWSS3, error) { conf := sendAWSS3Config{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform send_aws_s3: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_s3" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSS3{ @@ -67,7 +72,7 @@ func newSendAWSS3(_ context.Context, cfg config.Config) (*sendAWSS3, error) { Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -76,7 +81,7 @@ func newSendAWSS3(_ context.Context, cfg config.Config) (*sendAWSS3, error) { for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -116,7 +121,7 @@ func (tf *sendAWSS3) Transform(ctx context.Context, msg *message.Message) ([]*me } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -131,13 +136,13 @@ func (tf *sendAWSS3) Transform(ctx context.Context, msg *message.Message) ([]*me } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_s3: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_s3: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil @@ -167,7 +172,7 @@ func (tf *sendAWSS3) send(ctx context.Context, key string) error { data, err := withTransforms(ctx, tf.tforms, tf.agg.Get(key)) if err != nil { - return fmt.Errorf("transform: send_aws_s3: %v", err) + return err } for _, d := range data { diff --git a/transform/send_aws_sns.go b/transform/send_aws_sns.go index 6bb69a20..50ce7668 100644 --- a/transform/send_aws_sns.go +++ b/transform/send_aws_sns.go @@ -30,6 +30,7 @@ type sendAWSSNSConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -51,11 +52,15 @@ func (c *sendAWSSNSConfig) Validate() error { func newSendAWSSNS(_ context.Context, cfg config.Config) (*sendAWSSNS, error) { conf := sendAWSSNSConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_sns: %v", err) + return nil, fmt.Errorf("transform send_aws_sns: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_sns" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_sns: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendAWSSNS{ @@ -87,7 +92,7 @@ func newSendAWSSNS(_ context.Context, cfg config.Config) (*sendAWSSNS, error) { for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_sns: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -119,7 +124,7 @@ func (tf *sendAWSSNS) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_sns: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -128,7 +133,7 @@ func (tf *sendAWSSNS) Transform(ctx context.Context, msg *message.Message) ([]*m } if len(msg.Data()) > sendAWSSNSMessageSizeLimit { - return nil, fmt.Errorf("transform: send_aws_sns: %v", errSendAWSSNSMessageSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendAWSSNSMessageSizeLimit) } // If this value does not exist, then all data is batched together. @@ -138,13 +143,13 @@ func (tf *sendAWSSNS) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_sns: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_sns: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil } diff --git a/transform/send_aws_sqs.go b/transform/send_aws_sqs.go index 8fc44761..bcc664c2 100644 --- a/transform/send_aws_sqs.go +++ b/transform/send_aws_sqs.go @@ -31,6 +31,7 @@ type sendAWSSQSConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` AWS iconfig.AWS `json:"aws"` @@ -52,11 +53,15 @@ func (c *sendAWSSQSConfig) Validate() error { func newSendAWSSQS(_ context.Context, cfg config.Config) (*sendAWSSQS, error) { conf := sendAWSSQSConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", err) + return nil, fmt.Errorf("transform send_aws_sqs: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_aws_sqs" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } // arn:aws:sqs:region:account_id:queue_name @@ -96,7 +101,7 @@ func newSendAWSSQS(_ context.Context, cfg config.Config) (*sendAWSSQS, error) { for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -129,7 +134,7 @@ func (tf *sendAWSSQS) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -138,7 +143,7 @@ func (tf *sendAWSSQS) Transform(ctx context.Context, msg *message.Message) ([]*m } if len(msg.Data()) > sendSQSMessageSizeLimit { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", errSendSQSMessageSizeLimit) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendSQSMessageSizeLimit) } // If this value does not exist, then all data is batched together. @@ -148,13 +153,13 @@ func (tf *sendAWSSQS) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_aws_sqs: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil } diff --git a/transform/send_file.go b/transform/send_file.go index ca41c21e..23dd4ffd 100644 --- a/transform/send_file.go +++ b/transform/send_file.go @@ -24,6 +24,7 @@ type sendFileConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` } @@ -39,11 +40,15 @@ func (c *sendFileConfig) Validate() error { func newSendFile(_ context.Context, cfg config.Config) (*sendFile, error) { conf := sendFileConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform send_file: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_file" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendFile{ @@ -56,7 +61,7 @@ func newSendFile(_ context.Context, cfg config.Config) (*sendFile, error) { Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -65,7 +70,7 @@ func newSendFile(_ context.Context, cfg config.Config) (*sendFile, error) { for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -94,7 +99,7 @@ func (tf *sendFile) Transform(ctx context.Context, msg *message.Message) ([]*mes } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -109,13 +114,13 @@ func (tf *sendFile) Transform(ctx context.Context, msg *message.Message) ([]*mes } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_file: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_file: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil @@ -151,7 +156,7 @@ func (tf *sendFile) send(ctx context.Context, key string) error { data, err := withTransforms(ctx, tf.tforms, tf.agg.Get(key)) if err != nil { - return fmt.Errorf("transform: send_file: %v", err) + return err } for _, d := range data { diff --git a/transform/send_http_post.go b/transform/send_http_post.go index c202edfe..3fe42d5a 100644 --- a/transform/send_http_post.go +++ b/transform/send_http_post.go @@ -27,6 +27,7 @@ type sendHTTPPostConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` } @@ -46,11 +47,15 @@ func (c *sendHTTPPostConfig) Validate() error { func newSendHTTPPost(_ context.Context, cfg config.Config) (*sendHTTPPost, error) { conf := sendHTTPPostConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_http_post: %v", err) + return nil, fmt.Errorf("transform send_http_post: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_http_post" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: send_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := sendHTTPPost{ @@ -77,7 +82,7 @@ func newSendHTTPPost(_ context.Context, cfg config.Config) (*sendHTTPPost, error for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -109,7 +114,7 @@ func (tf *sendHTTPPost) Transform(ctx context.Context, msg *message.Message) ([] } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -124,13 +129,13 @@ func (tf *sendHTTPPost) Transform(ctx context.Context, msg *message.Message) ([] } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_http_post: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_http_post: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/send_stdout.go b/transform/send_stdout.go index 71529ba3..2c8f53fd 100644 --- a/transform/send_stdout.go +++ b/transform/send_stdout.go @@ -16,6 +16,7 @@ type sendStdoutConfig struct { // AuxTransforms are applied to batched data before it is sent. AuxTransforms []config.Config `json:"auxiliary_transforms"` + ID string `json:"id"` Object iconfig.Object `json:"object"` Batch iconfig.Batch `json:"batch"` } @@ -27,7 +28,11 @@ func (c *sendStdoutConfig) Decode(in interface{}) error { func newSendStdout(_ context.Context, cfg config.Config) (*sendStdout, error) { conf := sendStdoutConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: send_stdout: %v", err) + return nil, fmt.Errorf("transform send_stdout: %v", err) + } + + if conf.ID == "" { + conf.ID = "send_stdout" } tf := sendStdout{ @@ -40,7 +45,7 @@ func newSendStdout(_ context.Context, cfg config.Config) (*sendStdout, error) { Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: send_stdout: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.agg = agg @@ -49,7 +54,7 @@ func newSendStdout(_ context.Context, cfg config.Config) (*sendStdout, error) { for i, c := range conf.AuxTransforms { t, err := New(context.Background(), c) if err != nil { - return nil, fmt.Errorf("transform: send_stdout: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf.tforms[i] = t @@ -78,7 +83,7 @@ func (tf *sendStdout) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_stdout: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -93,13 +98,13 @@ func (tf *sendStdout) Transform(ctx context.Context, msg *message.Message) ([]*m } if err := tf.send(ctx, key); err != nil { - return nil, fmt.Errorf("transform: send_stdout: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } // If data cannot be added after reset, then the batch is misconfgured. tf.agg.Reset(key) if ok := tf.agg.Add(key, msg.Data()); !ok { - return nil, fmt.Errorf("transform: send_stdout: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } return []*message.Message{msg}, nil diff --git a/transform/string.go b/transform/string.go index 02c2c3a0..171e917f 100644 --- a/transform/string.go +++ b/transform/string.go @@ -8,6 +8,7 @@ import ( ) type strCaseConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } diff --git a/transform/string_append.go b/transform/string_append.go index e7b44a48..1af277c4 100644 --- a/transform/string_append.go +++ b/transform/string_append.go @@ -15,6 +15,7 @@ type stringAppendConfig struct { // Suffix is the string appended to the end of the string. Suffix string `json:"suffix"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -48,11 +49,15 @@ type stringAppend struct { func newStringAppend(_ context.Context, cfg config.Config) (*stringAppend, error) { conf := stringAppendConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_append: %v", err) + return nil, fmt.Errorf("transform string_append: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_append" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_append: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringAppend{ @@ -85,7 +90,7 @@ func (tf *stringAppend) Transform(ctx context.Context, msg *message.Message) ([] str := value.String() + tf.conf.Suffix if err := msg.SetValue(tf.conf.Object.TargetKey, str); err != nil { - return nil, fmt.Errorf("transform: string_append: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_capture.go b/transform/string_capture.go index 59fd876e..4d5d9cfe 100644 --- a/transform/string_capture.go +++ b/transform/string_capture.go @@ -25,6 +25,7 @@ type stringCaptureConfig struct { // is ignored. Count int `json:"count"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -58,11 +59,15 @@ func (c *stringCaptureConfig) Validate() error { func newStringCapture(_ context.Context, cfg config.Config) (*stringCapture, error) { conf := stringCaptureConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform string_capture: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_capture" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringCapture{ @@ -98,7 +103,7 @@ func (tf *stringCapture) Transform(_ context.Context, msg *message.Message) ([]* } if err := outMsg.SetValue(tf.conf.re.SubexpNames()[i], m); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -117,7 +122,7 @@ func (tf *stringCapture) Transform(_ context.Context, msg *message.Message) ([]* for _, s := range subs { m := strCaptureGetBytesMatch(s) if err := tmpMsg.SetValue("key.-1", m); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -149,7 +154,7 @@ func (tf *stringCapture) Transform(_ context.Context, msg *message.Message) ([]* // {"d":"e"} then the output is {"a":{"b":"c","d":"e"}}. setKey := tf.conf.Object.TargetKey + "." + tf.conf.re.SubexpNames()[i] if err := msg.SetValue(setKey, match); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -158,7 +163,7 @@ func (tf *stringCapture) Transform(_ context.Context, msg *message.Message) ([]* case tf.conf.Count == 0: matches := tf.conf.re.FindStringSubmatch(value.String()) if err := msg.SetValue(tf.conf.Object.TargetKey, strCaptureGetStringMatch(matches)); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil @@ -173,7 +178,7 @@ func (tf *stringCapture) Transform(_ context.Context, msg *message.Message) ([]* } if err := msg.SetValue(tf.conf.Object.TargetKey, matches); err != nil { - return nil, fmt.Errorf("transform: string_capture: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_replace.go b/transform/string_replace.go index 3c103373..6d2d3f1b 100644 --- a/transform/string_replace.go +++ b/transform/string_replace.go @@ -19,6 +19,7 @@ type stringReplaceConfig struct { // Replacement is the string to replace the matched values with. Replacement string `json:"replacement"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -52,11 +53,15 @@ func (c *stringReplaceConfig) Validate() error { func newStringReplace(_ context.Context, cfg config.Config) (*stringReplace, error) { conf := stringReplaceConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_replace: %v", err) + return nil, fmt.Errorf("transform string_replace: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_replace" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_replace: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringReplace{ @@ -94,7 +99,7 @@ func (tf *stringReplace) Transform(ctx context.Context, msg *message.Message) ([ s := tf.conf.re.ReplaceAllString(value.String(), string(tf.r)) if err := msg.SetValue(tf.conf.Object.TargetKey, s); err != nil { - return nil, fmt.Errorf("transform: string_replace: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_split.go b/transform/string_split.go index 5746ae68..330482c2 100644 --- a/transform/string_split.go +++ b/transform/string_split.go @@ -17,6 +17,7 @@ type stringSplitConfig struct { // Separator splits the string into elements of the array. Separator string `json:"separator"` + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -50,11 +51,15 @@ type stringSplit struct { func newStringSplit(_ context.Context, cfg config.Config) (*stringSplit, error) { conf := stringSplitConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_split: %v", err) + return nil, fmt.Errorf("transform string_split: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_split" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_split: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringSplit{ @@ -77,7 +82,7 @@ func (tf *stringSplit) Transform(ctx context.Context, msg *message.Message) ([]* b := bytes.Split(msg.Data(), tf.separator) for _, v := range b { if err := tmpMsg.SetValue("key.-1", v); err != nil { - return nil, fmt.Errorf("transform: string_split: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } @@ -95,7 +100,7 @@ func (tf *stringSplit) Transform(ctx context.Context, msg *message.Message) ([]* str := strings.Split(value.String(), tf.conf.Separator) if err := msg.SetValue(tf.conf.Object.TargetKey, str); err != nil { - return nil, fmt.Errorf("transform: string_split: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_to_lower.go b/transform/string_to_lower.go index 2d080787..19673794 100644 --- a/transform/string_to_lower.go +++ b/transform/string_to_lower.go @@ -14,11 +14,15 @@ import ( func newStringToLower(_ context.Context, cfg config.Config) (*stringToLower, error) { conf := strCaseConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform string_to_lower: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_to_lower" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringToLower{ @@ -53,7 +57,7 @@ func (tf *stringToLower) Transform(ctx context.Context, msg *message.Message) ([ s := strings.ToLower(value.String()) if err := msg.SetValue(tf.conf.Object.TargetKey, s); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_to_snake.go b/transform/string_to_snake.go index 2d0d867e..7683e591 100644 --- a/transform/string_to_snake.go +++ b/transform/string_to_snake.go @@ -13,11 +13,15 @@ import ( func newStringToSnake(_ context.Context, cfg config.Config) (*stringToSnake, error) { conf := strCaseConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_to_snake: %v", err) + return nil, fmt.Errorf("transform string_to_snake: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_to_snake" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_to_snake: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringToSnake{ @@ -52,7 +56,7 @@ func (tf *stringToSnake) Transform(ctx context.Context, msg *message.Message) ([ s := strcase.ToSnake(value.String()) if err := msg.SetValue(tf.conf.Object.TargetKey, s); err != nil { - return nil, fmt.Errorf("transform: string_to_snake: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_to_upper.go b/transform/string_to_upper.go index 7d2b158c..983cd1b7 100644 --- a/transform/string_to_upper.go +++ b/transform/string_to_upper.go @@ -14,11 +14,15 @@ import ( func newStringToUpper(_ context.Context, cfg config.Config) (*stringToUpper, error) { conf := strCaseConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform string_to_lower: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_to_upper" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := stringToUpper{ @@ -53,7 +57,7 @@ func (tf *stringToUpper) Transform(ctx context.Context, msg *message.Message) ([ s := strings.ToUpper(value.String()) if err := msg.SetValue(tf.conf.Object.TargetKey, s); err != nil { - return nil, fmt.Errorf("transform: string_to_lower: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/string_uuid.go b/transform/string_uuid.go index aa87bedf..59380bd2 100644 --- a/transform/string_uuid.go +++ b/transform/string_uuid.go @@ -12,6 +12,7 @@ import ( ) type stringUUIDConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -22,7 +23,11 @@ func (c *stringUUIDConfig) Decode(in interface{}) error { func newStringUUID(_ context.Context, cfg config.Config) (*stringUUID, error) { conf := stringUUIDConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: string_uuid: %v", err) + return nil, fmt.Errorf("transform string_uuid: %v", err) + } + + if conf.ID == "" { + conf.ID = "string_uuid" } tf := stringUUID{ @@ -46,7 +51,7 @@ func (tf *stringUUID) Transform(_ context.Context, msg *message.Message) ([]*mes uid := uuid.NewString() if tf.hasObjectSetKey { if err := msg.SetValue(tf.conf.Object.TargetKey, uid); err != nil { - return nil, fmt.Errorf("transform: string_uuid: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/time.go b/transform/time.go index 54edbcd9..a1a91c0f 100644 --- a/transform/time.go +++ b/transform/time.go @@ -13,6 +13,7 @@ const ( ) type timeUnixConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -33,6 +34,7 @@ func (c *timeUnixConfig) Validate() error { } type timePatternConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` Format string `json:"format"` diff --git a/transform/time_from_string.go b/transform/time_from_string.go index e3e2fa6d..4b3b14e0 100644 --- a/transform/time_from_string.go +++ b/transform/time_from_string.go @@ -12,11 +12,15 @@ import ( func newTimeFromString(_ context.Context, cfg config.Config) (*timeFromString, error) { conf := timePatternConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_from_stringing: %v", err) + return nil, fmt.Errorf("transform time_from_string: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_from_string" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_from_stringing: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeFromString{ @@ -50,12 +54,12 @@ func (tf *timeFromString) Transform(ctx context.Context, msg *message.Message) ( date, err := timeStrToUnix(value.String(), tf.conf.Format, tf.conf.Location) if err != nil { - return nil, fmt.Errorf("transform: time_from_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, date.UnixNano()); err != nil { - return nil, fmt.Errorf("transform: time_from_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value := timeUnixToBytes(date) diff --git a/transform/time_from_unix.go b/transform/time_from_unix.go index a4854a62..60fb5654 100644 --- a/transform/time_from_unix.go +++ b/transform/time_from_unix.go @@ -13,11 +13,15 @@ import ( func newTimeFromUnix(_ context.Context, cfg config.Config) (*timeFromUnix, error) { conf := timeUnixConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_from_unix: %v", err) + return nil, fmt.Errorf("transform time_from_unix: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_from_unix" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_from_unix: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeFromUnix{ @@ -55,7 +59,7 @@ func (tf *timeFromUnix) Transform(ctx context.Context, msg *message.Message) ([] if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, ns); err != nil { - return nil, fmt.Errorf("transform: time_from_unix: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value := []byte(fmt.Sprintf("%d", ns)) diff --git a/transform/time_from_unix_milli.go b/transform/time_from_unix_milli.go index 0deb2848..69191c48 100644 --- a/transform/time_from_unix_milli.go +++ b/transform/time_from_unix_milli.go @@ -13,11 +13,15 @@ import ( func newTimeFromUnixMilli(_ context.Context, cfg config.Config) (*timeFromUnixMilli, error) { conf := timeUnixConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_from_unix_milli: %v", err) + return nil, fmt.Errorf("transform time_from_unix_milli: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_from_unix_milli" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_from_unix_milli: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeFromUnixMilli{ @@ -55,7 +59,7 @@ func (tf *timeFromUnixMilli) Transform(ctx context.Context, msg *message.Message if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, ns); err != nil { - return nil, fmt.Errorf("transform: time_from_unix_milli: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value := []byte(fmt.Sprintf("%d", ns)) diff --git a/transform/time_now.go b/transform/time_now.go index 9c7d6c31..6eae9579 100644 --- a/transform/time_now.go +++ b/transform/time_now.go @@ -12,6 +12,7 @@ import ( ) type timeNowConfig struct { + ID string `json:"id"` Object iconfig.Object `json:"object"` } @@ -26,11 +27,15 @@ func (c *timeNowConfig) Validate() error { func newTimeNow(_ context.Context, cfg config.Config) (*timeNow, error) { conf := timeNowConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_now: %v", err) + return nil, fmt.Errorf("transform time_now: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_now" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_now: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeNow{ @@ -55,7 +60,7 @@ func (tf *timeNow) Transform(ctx context.Context, msg *message.Message) ([]*mess if tf.hasObjectSetKey { if err := msg.SetValue(tf.conf.Object.TargetKey, date.UnixNano()); err != nil { - return nil, fmt.Errorf("time: now: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } return []*message.Message{msg}, nil diff --git a/transform/time_to_string.go b/transform/time_to_string.go index c5c66646..eb2f3d89 100644 --- a/transform/time_to_string.go +++ b/transform/time_to_string.go @@ -12,11 +12,15 @@ import ( func newTimeToString(_ context.Context, cfg config.Config) (*timeToString, error) { conf := timePatternConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_to_string: %v", err) + return nil, fmt.Errorf("transform time_to_string: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_to_string" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeToString{ @@ -50,12 +54,12 @@ func (tf *timeToString) Transform(ctx context.Context, msg *message.Message) ([] pattern, err := timeUnixToStr(value.Int(), tf.conf.Format, tf.conf.Location) if err != nil { - return nil, fmt.Errorf("transform: time_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, pattern); err != nil { - return nil, fmt.Errorf("transform: time_to_string: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { msg.SetData([]byte(pattern)) diff --git a/transform/time_to_unix.go b/transform/time_to_unix.go index 0ba9003f..cd054be2 100644 --- a/transform/time_to_unix.go +++ b/transform/time_to_unix.go @@ -13,11 +13,15 @@ import ( func newTimeToUnix(_ context.Context, cfg config.Config) (*timeToUnix, error) { conf := timeUnixConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_to_unix: %v", err) + return nil, fmt.Errorf("transform time_to_unix: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_to_unix" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_to_unix: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeToUnix{ @@ -55,7 +59,7 @@ func (tf *timeToUnix) Transform(ctx context.Context, msg *message.Message) ([]*m if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, unix); err != nil { - return nil, fmt.Errorf("transform: time_to_unix: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value := []byte(fmt.Sprintf("%d", unix)) diff --git a/transform/time_to_unix_milli.go b/transform/time_to_unix_milli.go index 8c7a1c92..573b3fa1 100644 --- a/transform/time_to_unix_milli.go +++ b/transform/time_to_unix_milli.go @@ -13,11 +13,15 @@ import ( func newTimeToUnixMilli(_ context.Context, cfg config.Config) (*timeToUnixMilli, error) { conf := timeUnixConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: time_to_unix_milli: %v", err) + return nil, fmt.Errorf("transform time_to_unix_milli: %v", err) + } + + if conf.ID == "" { + conf.ID = "time_to_unix_milli" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: time_to_unix_milli: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := timeToUnixMilli{ @@ -55,7 +59,7 @@ func (tf *timeToUnixMilli) Transform(ctx context.Context, msg *message.Message) if tf.isObject { if err := msg.SetValue(tf.conf.Object.TargetKey, ms); err != nil { - return nil, fmt.Errorf("transform: time_to_unix_milli: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } } else { value := []byte(fmt.Sprintf("%d", ms)) diff --git a/transform/transform.go b/transform/transform.go index 00a73ab4..3f55f5de 100644 --- a/transform/transform.go +++ b/transform/transform.go @@ -193,7 +193,7 @@ func New(ctx context.Context, cfg config.Config) (Transformer, error) { //nolint case "utility_secret": return newUtilitySecret(ctx, cfg) default: - return nil, fmt.Errorf("transform: new: type %q settings %+v: %v", cfg.Type, cfg.Settings, errors.ErrInvalidFactoryInput) + return nil, fmt.Errorf("transform %s: %w", cfg.Type, errors.ErrInvalidFactoryInput) } } diff --git a/transform/utility_control.go b/transform/utility_control.go index adf6a6b7..4621b972 100644 --- a/transform/utility_control.go +++ b/transform/utility_control.go @@ -13,6 +13,7 @@ import ( ) type utilityControlConfig struct { + ID string `json:"id"` Batch iconfig.Batch `json:"batch"` } @@ -23,7 +24,11 @@ func (c *utilityControlConfig) Decode(in interface{}) error { func newUtilityControl(_ context.Context, cfg config.Config) (*utilityControl, error) { conf := utilityControlConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_control: %v", err) + return nil, fmt.Errorf("transform utility_control: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_control" } agg, err := aggregate.New(aggregate.Config{ @@ -32,7 +37,7 @@ func newUtilityControl(_ context.Context, cfg config.Config) (*utilityControl, e Duration: conf.Batch.Duration, }) if err != nil { - return nil, fmt.Errorf("transform: utility_control: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := utilityControl{ @@ -68,7 +73,7 @@ func (tf *utilityControl) Transform(_ context.Context, msg *message.Message) ([] tf.agg.Reset("") if ok := tf.agg.Add("", msg.Data()); !ok { - return nil, fmt.Errorf("transform: utility_control: %v", errSendBatchMisconfigured) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, errSendBatchMisconfigured) } ctrl := message.New().AsControl() diff --git a/transform/utility_delay.go b/transform/utility_delay.go index dfd0c33a..5793600e 100644 --- a/transform/utility_delay.go +++ b/transform/utility_delay.go @@ -15,6 +15,8 @@ import ( type utilityDelayConfig struct { // Duration is the amount of time to delay. Duration string `json:"duration"` + + ID string `json:"id"` } func (c *utilityDelayConfig) Decode(in interface{}) error { @@ -32,16 +34,20 @@ func (c *utilityDelayConfig) Validate() error { func newUtilityDelay(_ context.Context, cfg config.Config) (*utilityDelay, error) { conf := utilityDelayConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_delay: %v", err) + return nil, fmt.Errorf("transform utility_delay: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_delay" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: utility_delay: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } dur, err := time.ParseDuration(conf.Duration) if err != nil { - return nil, fmt.Errorf("transform: utility_delay: duration: %v", err) + return nil, fmt.Errorf("transform %s: duration: %v", conf.ID, err) } tf := utilityDelay{ diff --git a/transform/utility_drop.go b/transform/utility_drop.go index f940d4f1..24a888c0 100644 --- a/transform/utility_drop.go +++ b/transform/utility_drop.go @@ -10,7 +10,9 @@ import ( "github.com/brexhq/substation/message" ) -type utilityDropConfig struct{} +type utilityDropConfig struct { + ID string `json:"id"` +} func (c *utilityDropConfig) Decode(in interface{}) error { return iconfig.Decode(in, c) @@ -19,7 +21,11 @@ func (c *utilityDropConfig) Decode(in interface{}) error { func newUtilityDrop(_ context.Context, cfg config.Config) (*utilityDrop, error) { conf := utilityDropConfig{} if err := iconfig.Decode(cfg.Settings, &conf); err != nil { - return nil, fmt.Errorf("transform: utility_drop: %v", err) + return nil, fmt.Errorf("transform utility_drop: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_drop" } tf := utilityDrop{ diff --git a/transform/utility_err.go b/transform/utility_err.go index 9f98fd72..f7e10c11 100644 --- a/transform/utility_err.go +++ b/transform/utility_err.go @@ -13,6 +13,8 @@ import ( type utilityErrConfig struct { // Message is the error message to return. Message string `json:"message"` + + ID string `json:"id"` } func (c *utilityErrConfig) Decode(in interface{}) error { @@ -22,7 +24,11 @@ func (c *utilityErrConfig) Decode(in interface{}) error { func newUtilityErr(_ context.Context, cfg config.Config) (*utilityErr, error) { conf := utilityErrConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_err: %v", err) + return nil, fmt.Errorf("transform utility_err: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_err" } tf := utilityErr{ diff --git a/transform/utility_metric_bytes.go b/transform/utility_metric_bytes.go index 3668d3d4..8d91224e 100644 --- a/transform/utility_metric_bytes.go +++ b/transform/utility_metric_bytes.go @@ -14,6 +14,8 @@ import ( type utilityMetricBytesConfig struct { Metric iconfig.Metric `json:"metric"` + + ID string `json:"id"` } func (c *utilityMetricBytesConfig) Decode(in interface{}) error { @@ -24,12 +26,16 @@ func newUtilityMetricBytes(ctx context.Context, cfg config.Config) (*utilityMetr // conf gets validated when calling metrics.New. conf := utilityMetricBytesConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_metric_bytes: %v", err) + return nil, fmt.Errorf("transform utility_metric_bytes: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_metric_bytes" } m, err := metrics.New(ctx, conf.Metric.Destination) if err != nil { - return nil, fmt.Errorf("transform: utility_metric_bytes: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := utilityMetricBytes{ @@ -54,7 +60,7 @@ func (tf *utilityMetricBytes) Transform(ctx context.Context, msg *message.Messag Value: tf.bytes, Attributes: tf.conf.Metric.Attributes, }); err != nil { - return nil, fmt.Errorf("transform: utility_metric_bytes: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } atomic.StoreUint32(&tf.bytes, 0) diff --git a/transform/utility_metric_count.go b/transform/utility_metric_count.go index 79f141a7..1e283a30 100644 --- a/transform/utility_metric_count.go +++ b/transform/utility_metric_count.go @@ -14,6 +14,8 @@ import ( type utilityMetricsCountConfig struct { Metric iconfig.Metric `json:"metric"` + + ID string `json:"id"` } func (c *utilityMetricsCountConfig) Decode(in interface{}) error { @@ -24,12 +26,16 @@ func newUtilityMetricCount(ctx context.Context, cfg config.Config) (*utilityMetr // conf gets validated when calling metrics.New. conf := utilityMetricsCountConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_metric_count: %v", err) + return nil, fmt.Errorf("transform utility_metric_count: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_metric_count" } m, err := metrics.New(ctx, conf.Metric.Destination) if err != nil { - return nil, fmt.Errorf("transform: utility_metric_count: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := utilityMetricsCount{ @@ -54,7 +60,7 @@ func (tf *utilityMetricsCount) Transform(ctx context.Context, msg *message.Messa Value: tf.count, Attributes: tf.conf.Metric.Attributes, }); err != nil { - return nil, fmt.Errorf("transform: utility_metric_count: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } atomic.StoreUint32(&tf.count, 0) diff --git a/transform/utility_metric_freshness.go b/transform/utility_metric_freshness.go index 363c4ae6..8ac4f088 100644 --- a/transform/utility_metric_freshness.go +++ b/transform/utility_metric_freshness.go @@ -15,10 +15,11 @@ import ( ) type utilityMetricFreshnessConfig struct { - Threshold string `json:"threshold"` + Threshold string `json:"threshold"` + Metric iconfig.Metric `json:"metric"` + ID string `json:"id"` Object iconfig.Object `json:"object"` - Metric iconfig.Metric `json:"metric"` } func (c *utilityMetricFreshnessConfig) Decode(in interface{}) error { @@ -40,21 +41,25 @@ func (c *utilityMetricFreshnessConfig) Validate() error { func newUtilityMetricFreshness(ctx context.Context, cfg config.Config) (*utilityMetricFreshness, error) { conf := utilityMetricFreshnessConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: %v", err) + return nil, fmt.Errorf("transform utility_metric_freshness: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_metric_freshness" } if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } m, err := metrics.New(ctx, conf.Metric.Destination) if err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } dur, err := time.ParseDuration(conf.Threshold) if err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: duration: %v", err) + return nil, fmt.Errorf("transform %s: duration: %v", conf.ID, err) } tf := utilityMetricFreshness{ @@ -85,7 +90,7 @@ func (tf *utilityMetricFreshness) Transform(ctx context.Context, msg *message.Me Value: tf.success, Attributes: tf.conf.Metric.Attributes, }); err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } tf.conf.Metric.Attributes["FreshnessType"] = "Failure" @@ -94,7 +99,7 @@ func (tf *utilityMetricFreshness) Transform(ctx context.Context, msg *message.Me Value: tf.failure, Attributes: tf.conf.Metric.Attributes, }); err != nil { - return nil, fmt.Errorf("transform: utility_metric_freshness: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } atomic.StoreUint32(&tf.success, 0) diff --git a/transform/utility_secret.go b/transform/utility_secret.go index 179b6ed7..60903322 100644 --- a/transform/utility_secret.go +++ b/transform/utility_secret.go @@ -14,6 +14,8 @@ import ( type utilitySecretConfig struct { // Secret is the secret to retrieve. Secret config.Config `json:"secret"` + + ID string `json:"id"` } func (c *utilitySecretConfig) Decode(in interface{}) error { @@ -24,12 +26,16 @@ func newUtilitySecret(ctx context.Context, cfg config.Config) (*utilitySecret, e // conf gets validated when calling secrets.New. conf := utilitySecretConfig{} if err := conf.Decode(cfg.Settings); err != nil { - return nil, fmt.Errorf("transform: utility_secret: %v", err) + return nil, fmt.Errorf("transform utility_secret: %v", err) + } + + if conf.ID == "" { + conf.ID = "utility_secret" } ret, err := secrets.New(ctx, conf.Secret) if err != nil { - return nil, fmt.Errorf("transform: utility_secret: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } tf := utilitySecret{ @@ -38,7 +44,7 @@ func newUtilitySecret(ctx context.Context, cfg config.Config) (*utilitySecret, e } if err := tf.secret.Retrieve(ctx); err != nil { - return nil, fmt.Errorf("transform: utility_secret: %v", err) + return nil, fmt.Errorf("transform %s: %v", conf.ID, err) } return &tf, nil @@ -58,7 +64,7 @@ func (tf *utilitySecret) Transform(ctx context.Context, msg *message.Message) ([ if tf.secret.Expired() { if err := tf.secret.Retrieve(ctx); err != nil { - return nil, fmt.Errorf("transform: utility_secret: %v", err) + return nil, fmt.Errorf("transform %s: %v", tf.conf.ID, err) } }