From 0cf4f7e7c14f14883e43fb9ac0424e48dc3c3203 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Fri, 12 Feb 2021 15:23:36 -0500 Subject: [PATCH 01/11] initial chart for refinery --- refinery/.helmignore | 23 +++ refinery/Chart.yaml | 19 ++ refinery/config.toml | 237 +++++++++++++++++++++++ refinery/rules.toml | 231 ++++++++++++++++++++++ refinery/templates/NOTES.txt | 2 + refinery/templates/_helpers.tpl | 101 ++++++++++ refinery/templates/configmap-config.yaml | 9 + refinery/templates/configmap-rules.yaml | 9 + refinery/templates/deployment-redis.yaml | 49 +++++ refinery/templates/deployment.yaml | 86 ++++++++ refinery/templates/ingress.yaml | 41 ++++ refinery/templates/service-redis.yaml | 17 ++ refinery/templates/service.yaml | 22 +++ refinery/templates/serviceaccount.yaml | 12 ++ refinery/values.yaml | 125 ++++++++++++ 15 files changed, 983 insertions(+) create mode 100644 refinery/.helmignore create mode 100644 refinery/Chart.yaml create mode 100644 refinery/config.toml create mode 100644 refinery/rules.toml create mode 100644 refinery/templates/NOTES.txt create mode 100644 refinery/templates/_helpers.tpl create mode 100644 refinery/templates/configmap-config.yaml create mode 100644 refinery/templates/configmap-rules.yaml create mode 100644 refinery/templates/deployment-redis.yaml create mode 100644 refinery/templates/deployment.yaml create mode 100644 refinery/templates/ingress.yaml create mode 100644 refinery/templates/service-redis.yaml create mode 100644 refinery/templates/service.yaml create mode 100644 refinery/templates/serviceaccount.yaml create mode 100644 refinery/values.yaml diff --git a/refinery/.helmignore b/refinery/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/refinery/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/refinery/Chart.yaml b/refinery/Chart.yaml new file mode 100644 index 00000000..36a8ff04 --- /dev/null +++ b/refinery/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: refinery +description: Chart to deploy Honeycomb Refinery +type: application +version: 0.9.0 +appVersion: 0.16.0 +keywords: + - refinery + - honeycomb + - observability + - sampling + - tracing +home: https://honeycomb.io +sources: + - https://github.com/honeycombio/refinery +icon: https://www.honeycomb.io/wp-content/themes/honeycomb/assets/img/logo.svg +maintainers: + - name: puckpuck + email: pierre@honeycomb.io diff --git a/refinery/config.toml b/refinery/config.toml new file mode 100644 index 00000000..0ba4e7bb --- /dev/null +++ b/refinery/config.toml @@ -0,0 +1,237 @@ +##################### +## Refinery Config ## +##################### + +# ListenAddr is the IP and port on which to listen for incoming events. Incoming +# traffic is expected to be HTTP, so if using SSL put something like nginx in +# front to do the decryption. +# Should be of the form 0.0.0.0:8080 +# Not eligible for live reload. +ListenAddr = "{{ .Values.config.ListenAddr }}" + +# PeerListenAddr is the IP and port on which to listen for traffic being +# rerouted from a peer. Peer traffic is expected to be HTTP, so if using SSL +# put something like nginx in front to do the decryption. Must be different from +# ListenAddr +# Should be of the form 0.0.0.0:8081 +# Not eligible for live reload. +PeerListenAddr = "{{ .Values.config.PeerListenAddr }}" + +# APIKeys is a list of Honeycomb API keys that the proxy will accept. This list +# only applies to events - other Honeycomb API actions will fall through to the +# upstream API directly. +# Adding keys here causes events arriving with API keys not in this list to be +# rejected with an HTTP 401 error If an API key that is a literal '*' is in the +# list, all API keys are accepted. +# Eligible for live reload. +APIKeys = [ + # "replace-me", + # "more-optional-keys", + "*", # wildcard accept all keys + ] + +# HoneycombAPI is the URL for the upstream Honeycomb API. +# Eligible for live reload. +HoneycombAPI = "{{ .Values.config.HoneycombAPI }}" + +# SendDelay is a short timer that will be triggered when a trace is complete. +# Refinery will wait this duration before actually sending the trace. The +# reason for this short delay is to allow for small network delays or clock +# jitters to elapse and any final spans to arrive before actually sending the +# trace. This supports duration strings with supplied units. Set to 0 for +# immediate sends. +# Eligible for live reload. +SendDelay = "{{ .Values.config.SendDelay }}" + +# TraceTimeout is a long timer; it represents the outside boundary of how long +# to wait before sending an incomplete trace. Normally traces are sent when the +# root span arrives. Sometimes the root span never arrives (due to crashes or +# whatever), and this timer will send a trace even without having received the +# root span. If you have particularly long-lived traces you should increase this +# timer. This supports duration strings with supplied units. +# Eligible for live reload. +TraceTimeout = "{{ .Values.config.TraceTimeout }}" + +# SendTicker is a short timer; it determines the duration to use to check for traces to send +SendTicker = "{{ .Values.config.SendTicker }}" + +# LoggingLevel is the level above which we should log. Debug is very verbose, +# and should only be used in pre-production environments. Info is the +# recommended level. Valid options are "debug", "info", "error", and +# "panic" +# Not eligible for live reload. +LoggingLevel = "{{ .Values.config.LoggingLevel }}" + +# UpstreamBufferSize and PeerBufferSize control how large of an event queue to use +# when buffering events that will be forwarded to peers or the upstream API. +UpstreamBufferSize = {{ .Values.config.UpstreamBufferSize }} +PeerBufferSize = {{ .Values.config.PeerBufferSize }} + +# DebugServiceAddr sets the IP and port the debug service will run on +# The debug service will only run if the command line flag -d is specified +# The debug service runs on the first open port between localhost:6060 and :6069 by default +# DebugServiceAddr = "localhost:8085" + +############################ +## Implementation Choices ## +############################ + +# Each of the config options below chooses an implementation of a Refinery +# component to use. Depending on the choice there may be more configuration +# required below in the section for that choice. Changing implementation choices +# requires a process restart; these changes will not be picked up by a live +# config reload. (Individual config options for a given implementation may be +# eligible for live reload). + +# Collector describes which collector to use for collecting traces. The only +# current valid option is "InMemCollector".. More can be added by adding +# implementations of the Collector interface. +Collector = "InMemCollector" + +# Logger describes which logger to use for Refinery logs. Valid options are +# "logrus" and "honeycomb". The logrus option will write logs to STDOUT and the +# honeycomb option will send them to a Honeycomb dataset. +Logger = "{{ .Values.config.Logger }}" + +# Metrics describes which service to use for Refinery metrics. Valid options are +# "prometheus" and "honeycomb". The prometheus option starts a listener that +# will reply to a request for /metrics. The honeycomb option will send summary +# metrics to a Honeycomb dataset. +Metrics = "{{ .Values.config.Metrics }}" + +##################### +## Peer Management ## +##################### + +[PeerManagement] + +Type = "redis" + +# RedisHost is is used to connect to redis for peer cluster membership management. +# Further, if the environment variable 'REFINERY_REDIS_HOST' is set it takes +# precedence and this value is ignored. +# Not eligible for live reload. +RedisHost = "{{ if .Values.redis.existingHost }}{{ .Values.redis.existingHost | quote }}{{ else }}{{ include "refinery.redis.fullname" . }}:6379{{ end }}" + +# IdentifierInterfaceName is optional. By default, when using RedisHost, Refinery will use +# the local hostname to identify itself to other peers in Redis. If your environment +# requires that you use IPs as identifiers (for example, if peers can't resolve eachother +# by name), you can specify the network interface that Refinery is listening on here. +# Refinery will use the first unicast address that it finds on the specified network +# interface as its identifier. +# Not eligible for live reload. +IdentifierInterfaceName = "eth0" + +######################### +## In-Memory Collector ## +######################### + +# InMemCollector brings together all the settings that are relevant to +# collecting spans together to make traces. +[InMemCollector] + +# The collection cache is used to collect all spans into a trace as well as +# remember the sampling decision for any spans that might come in after the +# trace has been marked "complete" (either by timing out or seeing the root +# span). The number of traces in the cache should be many multiples (100x to +# 1000x) of the total number of concurrently active traces (trace throughput * +# trace duration). +# Eligible for live reload. Growing the cache capacity with a live config reload +# is fine. Avoid shrinking it with a live reload (you can, but it may cause +# temporary odd sampling decisions). +CacheCapacity = {{ .Values.config.InMemCollector.CacheCapacity }} + +# MaxAlloc is optional. If set, it must be an integer >= 0. 64-bit values are +# supported. +# If set to a non-zero value, once per tick (see SendTicker) the collector +# will compare total allocated bytes to this value. If allocation is too +# high, cache capacity will be reduced and an error will be logged. +# Useful values for this setting are generally in the range of 75%-90% of +# available system memory. +MaxAlloc = {{ .Values.config.InMemCollector.MaxAlloc }} + + +{{- if eq .Values.config.Logger "honeycomb" }} +###################### +## Honeycomb Logger ## +###################### + +# HoneycombLogger is a section of the config only used if you are using the +# HoneycombLogger to send all logs to a Honeycomb Dataset. If you are using a +# different logger (eg file-based logger) you can leave all this commented out. + +[HoneycombLogger] + +# LoggerHoneycombAPI is the URL for the upstream Honeycomb API. +# Eligible for live reload. +LoggerHoneycombAPI = "{{ .Values.config.HoneycombLogger.LoggerHoneycombAPI }}" + +# LoggerAPIKey is the API key to use to send log events to the Honeycomb logging +# dataset. This is separate from the APIKeys used to authenticate regular +# traffic. +# Eligible for live reload. +LoggerAPIKey = "{{ .Values.config.HoneycombLogger.LoggerAPIKey }}" + +# LoggerDataset is the name of the dataset to which to send Refinery logs +# Eligible for live reload. +LoggerDataset = "{{ .Values.config.HoneycombLogger.LoggerDataset }}" + +{{- else if eq .Values.config.Logger "logrus" }} +################### +## Logrus Logger ## +################### + +# LogrusLogger is a section of the config only used if you are using the +# LogrusLogger to send all logs to STDOUT using the logrus package. If you are +# using a different logger (eg honeycomb logger) you can leave all this +# commented out. +[LogrusLogger] + +# logrus logger currently has no options! +{{- end }} + + +{{- if eq .Values.config.Metrics "honeycomb" }} +####################### +## Honeycomb Metrics ## +####################### + +# HoneycombMetrics is a section of the config only used if you are using the +# HoneycombMetrics to send all metrics to a Honeycomb Dataset. If you are using a +# different metrics service (eg prometheus or metricsd) you can leave all this +# commented out. + +[HoneycombMetrics] + +# MetricsHoneycombAPI is the URL for the upstream Honeycomb API. +# Eligible for live reload. +MetricsHoneycombAPI = "{{ .Values.config.HoneycombMetrics.MetricsHoneycombAPI }}" + +# MetricsAPIKey is the API key to use to send log events to the Honeycomb logging +# dataset. This is separate from the APIKeys used to authenticate regular +# traffic. +# Eligible for live reload. +MetricsAPIKey = "{{ .Values.config.HoneycombMetrics.MetricsAPIKey }}" + +# MetricsDataset is the name of the dataset to which to send Refinery metrics +# Eligible for live reload. +MetricsDataset = "{{ .Values.config.HoneycombMetrics.MetricsDataset }}" + +# MetricsReportingInterval is the frequency (in seconds) to send metric events +# to Honeycomb. Between 1 and 60 is recommended. +# Not eligible for live reload. +MetricsReportingInterval = {{ .Values.config.HoneycombMetrics.MetricsReportingInterval }} + +{{- else if eq .Values.config.Metrics "prometheus" }} +######################## +## Prometheus Metrics ## +######################## + +[PrometheusMetrics] + +# MetricsListenAddr determines the interface and port on which Prometheus will +# listen for requests for /metrics. Must be different from the main Refinery +# listener. +# Not eligible for live reload. +MetricsListenAddr = "0.0.0.0:9090" +{{- end }} \ No newline at end of file diff --git a/refinery/rules.toml b/refinery/rules.toml new file mode 100644 index 00000000..65b43ad0 --- /dev/null +++ b/refinery/rules.toml @@ -0,0 +1,231 @@ +############################ +## Sampling Rules Config ## +############################ + +# DryRun - If enabled, marks traces that would be dropped given current sampling rules, +# and sends all traces regardless +DryRun = "{{ .Values.rules.DryRun }}" + +# DryRunFieldName - the key to add to use to add to event data when using DryRun mode above, defaults to refinery_kept +# DryRunFieldName = refinery_kept + +# DeterministicSampler is a section of the config for manipulating the +# Deterministic Sampler implementation. This is the simplest sampling algorithm +# - it is a static sample rate, choosing traces randomly to either keep or send +# (at the appropriate rate). It is not influenced by the contents of the trace. +Sampler = "{{ .Values.rules.Sampler }}" + +# SampleRate is the rate at which to sample. It indicates a ratio, where one +# sample trace is kept for every n traces seen. For example, a SampleRate of 30 +# will keep 1 out of every 30 traces. The choice on whether to keep any specific +# trace is random, so the rate is approximate. +# Eligible for live reload. +SampleRate = {{ .Values.rules.SampleRate }} + +[refinery-test] + + # Note: If your dataset name contains a space, you will have to escape the dataset name + # using single quotes, such as ['dataset 1'] + + # DynamicSampler is a section of the config for manipulating the simple Dynamic Sampler + # implementation. This sampler collects the values of a number of fields from a + # trace and uses them to form a key. This key is handed to the standard dynamic + # sampler algorithm which generates a sample rate based on the frequency with + # which that key has appeared in the previous ClearFrequencySec seconds. See + # https://github.com/honeycombio/dynsampler-go for more detail on the mechanics + # of the dynamic sampler. This sampler uses the AvgSampleRate algorithm from + # that package. + Sampler = "DeterministicSampler" + + # SampleRate is the goal rate at which to sample. It indicates a ratio, where + # one sample trace is kept for every n traces seen. For example, a SampleRate of + # 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic + # sampler, who assigns a sample rate for each trace based on the fields selected + # from that trace. + # Eligible for live reload. + SampleRate = 8 + + # FieldList is a list of all the field names to use to form the key that will be + # handed to the dynamic sampler. The cardinality of the combination of values + # from all of these keys should be reasonable in the face of the frequency of + # those keys. If the combination of fields in these keys essentially makes them + # unique, the dynamic sampler will do no sampling. If the keys have too few + # values, you won't get samples of the most interesting traces. A good key + # selection will have consistent values for high frequency boring traffic and + # unique values for outliers and interesting traffic. Including an error field + # (or something like HTTP status code) is an excellent choice. As an example, + # assuming 30 or so endpoints, a combination of HTTP endpoint and status code + # would be a good set of keys in order to let you see accurately use of all + # endpoints and call out when there is failing traffic to any endpoint. Field + # names may come from any span in the trace. + # Eligible for live reload. + FieldList = ["name", "client"] + + # UseTraceLength will add the number of spans in the trace in to the dynamic + # sampler as part of the key. The number of spans is exact, so if there are + # normally small variations in trace length you may want to leave this off. If + # traces are consistent lengths and changes in trace length is a useful + # indicator of traces you'd like to see in Honeycomb, set this to true. + # Eligible for live reload. + UseTraceLength = true + + # AddSampleRateKeyToTrace when this is set to true, the sampler will add a field + # to the root span of the trace containing the key used by the sampler to decide + # the sample rate. This can be helpful in understanding why the sampler is + # making certain decisions about sample rate and help you understand how to + # better choose the sample rate key (aka the FieldList setting above) to use. + AddSampleRateKeyToTrace = true + + # AddSampleRateKeyToTraceField is the name of the field the sampler will use + # when adding the sample rate key to the trace. This setting is only used when + # AddSampleRateKeyToTrace is true. + AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key" + + # ClearFrequencySec is the name of the field the sampler will use to determine + # the period over which it will calculate the sample rate. This setting defaults + # to 30. + # Eligible for live reload. + ClearFrequencySec = 60 + +[dataset2] + + # EMADynamicSampler is a section of the config for manipulating the Exponential + # Moving Average (EMA) Dynamic Sampler implementation. Like the simple DynamicSampler, + # it attempts to average a given sample rate, weighting rare traffic and frequent + # traffic differently so as to end up with the correct average. + # + # EMADynamicSampler is an improvement upon the simple DynamicSampler and is recommended + # for most use cases. Based on the DynamicSampler implementation, EMADynamicSampler differs + # in that rather than compute rate based on a periodic sample of traffic, it maintains an Exponential + # Moving Average of counts seen per key, and adjusts this average at regular intervals. + # The weight applied to more recent intervals is defined by `weight`, a number between + # (0, 1) - larger values weight the average more toward recent observations. In other words, + # a larger weight will cause sample rates more quickly adapt to traffic patterns, + # while a smaller weight will result in sample rates that are less sensitive to bursts or drops + # in traffic and thus more consistent over time. + # + # Keys that are not found in the EMA will always have a sample + # rate of 1. Keys that occur more frequently will be sampled on a logarithmic + # curve. In other words, every key will be represented at least once in any + # given window and more frequent keys will have their sample rate + # increased proportionally to wind up with the goal sample rate. + Sampler = "EMADynamicSampler" + + # GoalSampleRate is the goal rate at which to sample. It indicates a ratio, where + # one sample trace is kept for every n traces seen. For example, a SampleRate of + # 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic + # sampler, who assigns a sample rate for each trace based on the fields selected + # from that trace. + # Eligible for live reload. + GoalSampleRate = 2 + + # FieldList is a list of all the field names to use to form the key that will be + # handed to the dynamic sampler. The cardinality of the combination of values + # from all of these keys should be reasonable in the face of the frequency of + # those keys. If the combination of fields in these keys essentially makes them + # unique, the dynamic sampler will do no sampling. If the keys have too few + # values, you won't get samples of the most interesting traces. A good key + # selection will have consistent values for high frequency boring traffic and + # unique values for outliers and interesting traffic. Including an error field + # (or something like HTTP status code) is an excellent choice. As an example, + # assuming 30 or so endpoints, a combination of HTTP endpoint and status code + # would be a good set of keys in order to let you see accurately use of all + # endpoints and call out when there is failing traffic to any endpoint. Field + # names may come from any span in the trace. + # Eligible for live reload. + FieldList = ["request.method","response.status_code"] + + # UseTraceLength will add the number of spans in the trace in to the dynamic + # sampler as part of the key. The number of spans is exact, so if there are + # normally small variations in trace length you may want to leave this off. If + # traces are consistent lengths and changes in trace length is a useful + # indicator of traces you'd like to see in Honeycomb, set this to true. + # Eligible for live reload. + UseTraceLength = true + + # AddSampleRateKeyToTrace when this is set to true, the sampler will add a field + # to the root span of the trace containing the key used by the sampler to decide + # the sample rate. This can be helpful in understanding why the sampler is + # making certain decisions about sample rate and help you understand how to + # better choose the sample rate key (aka the FieldList setting above) to use. + AddSampleRateKeyToTrace = true + + # AddSampleRateKeyToTraceField is the name of the field the sampler will use + # when adding the sample rate key to the trace. This setting is only used when + # AddSampleRateKeyToTrace is true. + AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key" + + # AdjustmentInterval defines how often (in seconds) we adjust the moving average from + # recent observations. Default 15s + # Eligible for live reload. + AdjustmentInterval = 15 + + # Weight is a value between (0, 1) indicating the weighting factor used to adjust + # the EMA. With larger values, newer data will influence the average more, and older + # values will be factored out more quickly. In mathematical literature concerning EMA, + # this is referred to as the `alpha` constant. + # Default is 0.5 + # Eligible for live reload. + Weight = 0.5 + + # MaxKeys, if greater than 0, limits the number of distinct keys tracked in EMA. + # Once MaxKeys is reached, new keys will not be included in the sample rate map, but + # existing keys will continue to be be counted. You can use this to keep the sample rate + # map size under control. + # Eligible for live reload + MaxKeys = 0 + + # AgeOutValue indicates the threshold for removing keys from the EMA. The EMA of any key + # will approach 0 if it is not repeatedly observed, but will never truly reach it, so we have to + # decide what constitutes "zero". Keys with averages below this threshold will be removed + # from the EMA. Default is the same as Weight, as this prevents a key with the smallest + # integer value (1) from being aged out immediately. This value should generally be <= Weight, + # unless you have very specific reasons to set it higher. + # Eligible for live reload + AgeOutValue = 0.5 + + # BurstMultiple, if set, is multiplied by the sum of the running average of counts to define + # the burst detection threshold. If total counts observed for a given interval exceed the threshold + # EMA is updated immediately, rather than waiting on the AdjustmentInterval. + # Defaults to 2; negative value disables. With a default of 2, if your traffic suddenly doubles, + # burst detection will kick in. + # Eligible for live reload + BurstMultiple = 2.0 + + # BurstDetectionDelay indicates the number of intervals to run after Start is called before + # burst detection kicks in. + # Defaults to 3 + # Eligible for live reload + BurstDetectionDelay = 3 + +[dataset3] + + Sampler = "DeterministicSampler" + SampleRate = 10 + +[dataset4] + + Sampler = "RulesBasedSampler" + + [[dataset4.rule]] + name = "500 errors" + SampleRate = 1 + [[dataset4.rule.condition]] + field = "status_code" + operator = "=" + value = 500 + [[dataset4.rule.condition]] + field = "duration_ms" + operator = ">=" + value = 1000.789 + + [[dataset4.rule]] + name = "drop 200 responses" + drop = true + [[dataset4.rule.condition]] + field = "status_code" + operator = "=" + value = 200 + + [[dataset4.rule]] + SampleRate = 10 # default when no rules match, if missing defaults to 1 diff --git a/refinery/templates/NOTES.txt b/refinery/templates/NOTES.txt new file mode 100644 index 00000000..ff2d5a47 --- /dev/null +++ b/refinery/templates/NOTES.txt @@ -0,0 +1,2 @@ +Honeycomb refinery is setup and configured to refine events that are sent through it. You should see data flowing +within a few minutes at https://ui.honeycomb.io diff --git a/refinery/templates/_helpers.tpl b/refinery/templates/_helpers.tpl new file mode 100644 index 00000000..a01dd237 --- /dev/null +++ b/refinery/templates/_helpers.tpl @@ -0,0 +1,101 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "refinery.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "refinery.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "refinery.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "refinery.labels" -}} +helm.sh/chart: {{ include "refinery.chart" . }} +{{ include "refinery.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "refinery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "refinery.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + + +{{/* +Create a default fully qualified app name for the redis component. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "refinery.redis.fullname" -}} +{{ include "refinery.fullname" . }}-redis +{{- end }} + + +{{/* +Common labels for redis +*/}} +{{- define "refinery.redis.labels" -}} +helm.sh/chart: {{ include "refinery.chart" . }} +{{ include "refinery.redis.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels for redis +*/}} +{{- define "refinery.redis.selectorLabels" -}} +app.kubernetes.io/name: {{ include "refinery.name" . }}-redis +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + + + + + + + +{{/* +Create the name of the service account to use +*/}} +{{- define "refinery.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "refinery.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/refinery/templates/configmap-config.yaml b/refinery/templates/configmap-config.yaml new file mode 100644 index 00000000..8ee316a2 --- /dev/null +++ b/refinery/templates/configmap-config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "refinery.fullname" . }}-config + labels: + {{- include "refinery.labels" . | nindent 4 }} +data: + config.toml: | +{{ tpl (.Files.Get "config.toml") . | indent 6 }} diff --git a/refinery/templates/configmap-rules.yaml b/refinery/templates/configmap-rules.yaml new file mode 100644 index 00000000..e18fd1af --- /dev/null +++ b/refinery/templates/configmap-rules.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "refinery.fullname" . }}-rules + labels: + {{- include "refinery.labels" . | nindent 4 }} +data: + rules.toml: |- +{{ tpl (.Files.Get "rules.toml") . | indent 6 }} diff --git a/refinery/templates/deployment-redis.yaml b/refinery/templates/deployment-redis.yaml new file mode 100644 index 00000000..5d23c399 --- /dev/null +++ b/refinery/templates/deployment-redis.yaml @@ -0,0 +1,49 @@ +{{- if .Values.redis.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "refinery.redis.fullname" . }} + labels: + {{- include "refinery.redis.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "refinery.redis.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "refinery.redis.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "refinery.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: redis + image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" + imagePullPolicy: {{ .Values.redis.image.pullPolicy }} + ports: + - name: http + containerPort: 6379 + protocol: TCP + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/refinery/templates/deployment.yaml b/refinery/templates/deployment.yaml new file mode 100644 index 00000000..19c8701d --- /dev/null +++ b/refinery/templates/deployment.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "refinery.fullname" . }} + labels: + {{- include "refinery.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "refinery.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if eq .Values.config.Metrics "prometheus" }} + prometheus.io/port: "9090" + prometheus.io/scrape: "true" + {{- end }} + labels: + {{- include "refinery.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "refinery.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["refinery"] + ports: + - name: data + containerPort: 8080 + protocol: TCP + - name: peer + containerPort: 8081 + protocol: TCP + {{- if eq .Values.config.Metrics "prometheus" }} + - name: metrics + containerPort: 9090 + protocol: TCP + {{- end }} + volumeMounts: + - name: refinery-config + mountPath: /etc/refinery/ + livenessProbe: + httpGet: + path: /alive + port: incoming + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: refinery-config + projected: + sources: + - configMap: + name: {{ include "refinery.fullname" . }}-config + items: + - key: config.toml + path: refinery.toml + - configMap: + name: {{ include "refinery.fullname" . }}-rules + items: + - key: rules.toml + path: rules.toml + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/refinery/templates/ingress.yaml b/refinery/templates/ingress.yaml new file mode 100644 index 00000000..a8c68b52 --- /dev/null +++ b/refinery/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "refinery.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "refinery.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} diff --git a/refinery/templates/service-redis.yaml b/refinery/templates/service-redis.yaml new file mode 100644 index 00000000..198b975a --- /dev/null +++ b/refinery/templates/service-redis.yaml @@ -0,0 +1,17 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "refinery.redis.fullname" . }} + labels: + {{- include "refinery.redis.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 6379 + protocol: TCP + targetPort: 6379 + selector: + {{- include "refinery.redis.selectorLabels" . | nindent 4 }} +{{- end}} \ No newline at end of file diff --git a/refinery/templates/service.yaml b/refinery/templates/service.yaml new file mode 100644 index 00000000..8df5bf90 --- /dev/null +++ b/refinery/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "refinery.fullname" . }} + labels: + {{- include "refinery.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: data + protocol: TCP + name: data + {{- if eq .Values.config.Metrics "prometheus" }} + - port: 9090 + targetPort: metrics + protocol: TCP + name: metrics + {{- end }} + + selector: + {{- include "refinery.selectorLabels" . | nindent 4 }} diff --git a/refinery/templates/serviceaccount.yaml b/refinery/templates/serviceaccount.yaml new file mode 100644 index 00000000..2b2b9cd7 --- /dev/null +++ b/refinery/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "refinery.serviceAccountName" . }} + labels: + {{- include "refinery.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/refinery/values.yaml b/refinery/values.yaml new file mode 100644 index 00000000..57a48e5b --- /dev/null +++ b/refinery/values.yaml @@ -0,0 +1,125 @@ +# Default values for refinery. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 2 + +image: + repository: honeycombio/refinery + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + + +# Values used to build config.toml +config: + ListenAddr: "0.0.0.0:8080" + PeerListenAddr: "0.0.0.0:8081" + HoneycombAPI: "https://api.honeycomb.io" + LoggingLevel: error + SendDelay: "2s" + TraceTimeout: "300s" + SendTicker: "100ms" + UpstreamBufferSize: 1000 + PeerBufferSize: 1000 + + # Implementation Choices + Logger: "logrus" + Metrics: "prometheus" + + InMemCollector: + CacheCapacity: 1000 + MaxAlloc: 0 + + HoneycombLogger: + LoggerHoneycombAPI: "https://api.honeycomb.io" + LoggerAPIKey: "YOUR_API_KEY" + LoggerDataset: "refinery-logs" + + HoneycombMetrics: + MetricsHoneycombAPI: "https://api.honeycomb.io" + MetricsAPIKey: "YOUR_API_KEY" + MetricsDataset: "refinery-metrics" + MetricsReportingInterval: 3 + +# Values used to configure rules.toml with the same name +rules: + DryRun: false + Sampler: DeterministicSampler + SampleRate: 1 + +redis: + enabled: true + # existingHost: + image: + repository: redis + tag: 6.0.2 + pullPolicy: IfNotPresent + + +# Creates service monitor to collect prom metrics (depends on kube-prom) +# prometheus: +# serviceMonitor: +# create: true +# namespace: monitoring +prometheus: + serviceMonitor: + create: false + + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 500Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 6c166dc593886fd4619e1536bd908b79a464da8a Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Wed, 24 Feb 2021 14:01:08 -0500 Subject: [PATCH 02/11] move to yaml based configs --- refinery/config.toml | 237 ----------------------- refinery/rules.toml | 231 ---------------------- refinery/templates/configmap-config.yaml | 4 +- refinery/templates/configmap-rules.yaml | 4 +- refinery/templates/deployment.yaml | 15 +- refinery/values.yaml | 50 ++--- 6 files changed, 42 insertions(+), 499 deletions(-) delete mode 100644 refinery/config.toml delete mode 100644 refinery/rules.toml diff --git a/refinery/config.toml b/refinery/config.toml deleted file mode 100644 index 0ba4e7bb..00000000 --- a/refinery/config.toml +++ /dev/null @@ -1,237 +0,0 @@ -##################### -## Refinery Config ## -##################### - -# ListenAddr is the IP and port on which to listen for incoming events. Incoming -# traffic is expected to be HTTP, so if using SSL put something like nginx in -# front to do the decryption. -# Should be of the form 0.0.0.0:8080 -# Not eligible for live reload. -ListenAddr = "{{ .Values.config.ListenAddr }}" - -# PeerListenAddr is the IP and port on which to listen for traffic being -# rerouted from a peer. Peer traffic is expected to be HTTP, so if using SSL -# put something like nginx in front to do the decryption. Must be different from -# ListenAddr -# Should be of the form 0.0.0.0:8081 -# Not eligible for live reload. -PeerListenAddr = "{{ .Values.config.PeerListenAddr }}" - -# APIKeys is a list of Honeycomb API keys that the proxy will accept. This list -# only applies to events - other Honeycomb API actions will fall through to the -# upstream API directly. -# Adding keys here causes events arriving with API keys not in this list to be -# rejected with an HTTP 401 error If an API key that is a literal '*' is in the -# list, all API keys are accepted. -# Eligible for live reload. -APIKeys = [ - # "replace-me", - # "more-optional-keys", - "*", # wildcard accept all keys - ] - -# HoneycombAPI is the URL for the upstream Honeycomb API. -# Eligible for live reload. -HoneycombAPI = "{{ .Values.config.HoneycombAPI }}" - -# SendDelay is a short timer that will be triggered when a trace is complete. -# Refinery will wait this duration before actually sending the trace. The -# reason for this short delay is to allow for small network delays or clock -# jitters to elapse and any final spans to arrive before actually sending the -# trace. This supports duration strings with supplied units. Set to 0 for -# immediate sends. -# Eligible for live reload. -SendDelay = "{{ .Values.config.SendDelay }}" - -# TraceTimeout is a long timer; it represents the outside boundary of how long -# to wait before sending an incomplete trace. Normally traces are sent when the -# root span arrives. Sometimes the root span never arrives (due to crashes or -# whatever), and this timer will send a trace even without having received the -# root span. If you have particularly long-lived traces you should increase this -# timer. This supports duration strings with supplied units. -# Eligible for live reload. -TraceTimeout = "{{ .Values.config.TraceTimeout }}" - -# SendTicker is a short timer; it determines the duration to use to check for traces to send -SendTicker = "{{ .Values.config.SendTicker }}" - -# LoggingLevel is the level above which we should log. Debug is very verbose, -# and should only be used in pre-production environments. Info is the -# recommended level. Valid options are "debug", "info", "error", and -# "panic" -# Not eligible for live reload. -LoggingLevel = "{{ .Values.config.LoggingLevel }}" - -# UpstreamBufferSize and PeerBufferSize control how large of an event queue to use -# when buffering events that will be forwarded to peers or the upstream API. -UpstreamBufferSize = {{ .Values.config.UpstreamBufferSize }} -PeerBufferSize = {{ .Values.config.PeerBufferSize }} - -# DebugServiceAddr sets the IP and port the debug service will run on -# The debug service will only run if the command line flag -d is specified -# The debug service runs on the first open port between localhost:6060 and :6069 by default -# DebugServiceAddr = "localhost:8085" - -############################ -## Implementation Choices ## -############################ - -# Each of the config options below chooses an implementation of a Refinery -# component to use. Depending on the choice there may be more configuration -# required below in the section for that choice. Changing implementation choices -# requires a process restart; these changes will not be picked up by a live -# config reload. (Individual config options for a given implementation may be -# eligible for live reload). - -# Collector describes which collector to use for collecting traces. The only -# current valid option is "InMemCollector".. More can be added by adding -# implementations of the Collector interface. -Collector = "InMemCollector" - -# Logger describes which logger to use for Refinery logs. Valid options are -# "logrus" and "honeycomb". The logrus option will write logs to STDOUT and the -# honeycomb option will send them to a Honeycomb dataset. -Logger = "{{ .Values.config.Logger }}" - -# Metrics describes which service to use for Refinery metrics. Valid options are -# "prometheus" and "honeycomb". The prometheus option starts a listener that -# will reply to a request for /metrics. The honeycomb option will send summary -# metrics to a Honeycomb dataset. -Metrics = "{{ .Values.config.Metrics }}" - -##################### -## Peer Management ## -##################### - -[PeerManagement] - -Type = "redis" - -# RedisHost is is used to connect to redis for peer cluster membership management. -# Further, if the environment variable 'REFINERY_REDIS_HOST' is set it takes -# precedence and this value is ignored. -# Not eligible for live reload. -RedisHost = "{{ if .Values.redis.existingHost }}{{ .Values.redis.existingHost | quote }}{{ else }}{{ include "refinery.redis.fullname" . }}:6379{{ end }}" - -# IdentifierInterfaceName is optional. By default, when using RedisHost, Refinery will use -# the local hostname to identify itself to other peers in Redis. If your environment -# requires that you use IPs as identifiers (for example, if peers can't resolve eachother -# by name), you can specify the network interface that Refinery is listening on here. -# Refinery will use the first unicast address that it finds on the specified network -# interface as its identifier. -# Not eligible for live reload. -IdentifierInterfaceName = "eth0" - -######################### -## In-Memory Collector ## -######################### - -# InMemCollector brings together all the settings that are relevant to -# collecting spans together to make traces. -[InMemCollector] - -# The collection cache is used to collect all spans into a trace as well as -# remember the sampling decision for any spans that might come in after the -# trace has been marked "complete" (either by timing out or seeing the root -# span). The number of traces in the cache should be many multiples (100x to -# 1000x) of the total number of concurrently active traces (trace throughput * -# trace duration). -# Eligible for live reload. Growing the cache capacity with a live config reload -# is fine. Avoid shrinking it with a live reload (you can, but it may cause -# temporary odd sampling decisions). -CacheCapacity = {{ .Values.config.InMemCollector.CacheCapacity }} - -# MaxAlloc is optional. If set, it must be an integer >= 0. 64-bit values are -# supported. -# If set to a non-zero value, once per tick (see SendTicker) the collector -# will compare total allocated bytes to this value. If allocation is too -# high, cache capacity will be reduced and an error will be logged. -# Useful values for this setting are generally in the range of 75%-90% of -# available system memory. -MaxAlloc = {{ .Values.config.InMemCollector.MaxAlloc }} - - -{{- if eq .Values.config.Logger "honeycomb" }} -###################### -## Honeycomb Logger ## -###################### - -# HoneycombLogger is a section of the config only used if you are using the -# HoneycombLogger to send all logs to a Honeycomb Dataset. If you are using a -# different logger (eg file-based logger) you can leave all this commented out. - -[HoneycombLogger] - -# LoggerHoneycombAPI is the URL for the upstream Honeycomb API. -# Eligible for live reload. -LoggerHoneycombAPI = "{{ .Values.config.HoneycombLogger.LoggerHoneycombAPI }}" - -# LoggerAPIKey is the API key to use to send log events to the Honeycomb logging -# dataset. This is separate from the APIKeys used to authenticate regular -# traffic. -# Eligible for live reload. -LoggerAPIKey = "{{ .Values.config.HoneycombLogger.LoggerAPIKey }}" - -# LoggerDataset is the name of the dataset to which to send Refinery logs -# Eligible for live reload. -LoggerDataset = "{{ .Values.config.HoneycombLogger.LoggerDataset }}" - -{{- else if eq .Values.config.Logger "logrus" }} -################### -## Logrus Logger ## -################### - -# LogrusLogger is a section of the config only used if you are using the -# LogrusLogger to send all logs to STDOUT using the logrus package. If you are -# using a different logger (eg honeycomb logger) you can leave all this -# commented out. -[LogrusLogger] - -# logrus logger currently has no options! -{{- end }} - - -{{- if eq .Values.config.Metrics "honeycomb" }} -####################### -## Honeycomb Metrics ## -####################### - -# HoneycombMetrics is a section of the config only used if you are using the -# HoneycombMetrics to send all metrics to a Honeycomb Dataset. If you are using a -# different metrics service (eg prometheus or metricsd) you can leave all this -# commented out. - -[HoneycombMetrics] - -# MetricsHoneycombAPI is the URL for the upstream Honeycomb API. -# Eligible for live reload. -MetricsHoneycombAPI = "{{ .Values.config.HoneycombMetrics.MetricsHoneycombAPI }}" - -# MetricsAPIKey is the API key to use to send log events to the Honeycomb logging -# dataset. This is separate from the APIKeys used to authenticate regular -# traffic. -# Eligible for live reload. -MetricsAPIKey = "{{ .Values.config.HoneycombMetrics.MetricsAPIKey }}" - -# MetricsDataset is the name of the dataset to which to send Refinery metrics -# Eligible for live reload. -MetricsDataset = "{{ .Values.config.HoneycombMetrics.MetricsDataset }}" - -# MetricsReportingInterval is the frequency (in seconds) to send metric events -# to Honeycomb. Between 1 and 60 is recommended. -# Not eligible for live reload. -MetricsReportingInterval = {{ .Values.config.HoneycombMetrics.MetricsReportingInterval }} - -{{- else if eq .Values.config.Metrics "prometheus" }} -######################## -## Prometheus Metrics ## -######################## - -[PrometheusMetrics] - -# MetricsListenAddr determines the interface and port on which Prometheus will -# listen for requests for /metrics. Must be different from the main Refinery -# listener. -# Not eligible for live reload. -MetricsListenAddr = "0.0.0.0:9090" -{{- end }} \ No newline at end of file diff --git a/refinery/rules.toml b/refinery/rules.toml deleted file mode 100644 index 65b43ad0..00000000 --- a/refinery/rules.toml +++ /dev/null @@ -1,231 +0,0 @@ -############################ -## Sampling Rules Config ## -############################ - -# DryRun - If enabled, marks traces that would be dropped given current sampling rules, -# and sends all traces regardless -DryRun = "{{ .Values.rules.DryRun }}" - -# DryRunFieldName - the key to add to use to add to event data when using DryRun mode above, defaults to refinery_kept -# DryRunFieldName = refinery_kept - -# DeterministicSampler is a section of the config for manipulating the -# Deterministic Sampler implementation. This is the simplest sampling algorithm -# - it is a static sample rate, choosing traces randomly to either keep or send -# (at the appropriate rate). It is not influenced by the contents of the trace. -Sampler = "{{ .Values.rules.Sampler }}" - -# SampleRate is the rate at which to sample. It indicates a ratio, where one -# sample trace is kept for every n traces seen. For example, a SampleRate of 30 -# will keep 1 out of every 30 traces. The choice on whether to keep any specific -# trace is random, so the rate is approximate. -# Eligible for live reload. -SampleRate = {{ .Values.rules.SampleRate }} - -[refinery-test] - - # Note: If your dataset name contains a space, you will have to escape the dataset name - # using single quotes, such as ['dataset 1'] - - # DynamicSampler is a section of the config for manipulating the simple Dynamic Sampler - # implementation. This sampler collects the values of a number of fields from a - # trace and uses them to form a key. This key is handed to the standard dynamic - # sampler algorithm which generates a sample rate based on the frequency with - # which that key has appeared in the previous ClearFrequencySec seconds. See - # https://github.com/honeycombio/dynsampler-go for more detail on the mechanics - # of the dynamic sampler. This sampler uses the AvgSampleRate algorithm from - # that package. - Sampler = "DeterministicSampler" - - # SampleRate is the goal rate at which to sample. It indicates a ratio, where - # one sample trace is kept for every n traces seen. For example, a SampleRate of - # 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic - # sampler, who assigns a sample rate for each trace based on the fields selected - # from that trace. - # Eligible for live reload. - SampleRate = 8 - - # FieldList is a list of all the field names to use to form the key that will be - # handed to the dynamic sampler. The cardinality of the combination of values - # from all of these keys should be reasonable in the face of the frequency of - # those keys. If the combination of fields in these keys essentially makes them - # unique, the dynamic sampler will do no sampling. If the keys have too few - # values, you won't get samples of the most interesting traces. A good key - # selection will have consistent values for high frequency boring traffic and - # unique values for outliers and interesting traffic. Including an error field - # (or something like HTTP status code) is an excellent choice. As an example, - # assuming 30 or so endpoints, a combination of HTTP endpoint and status code - # would be a good set of keys in order to let you see accurately use of all - # endpoints and call out when there is failing traffic to any endpoint. Field - # names may come from any span in the trace. - # Eligible for live reload. - FieldList = ["name", "client"] - - # UseTraceLength will add the number of spans in the trace in to the dynamic - # sampler as part of the key. The number of spans is exact, so if there are - # normally small variations in trace length you may want to leave this off. If - # traces are consistent lengths and changes in trace length is a useful - # indicator of traces you'd like to see in Honeycomb, set this to true. - # Eligible for live reload. - UseTraceLength = true - - # AddSampleRateKeyToTrace when this is set to true, the sampler will add a field - # to the root span of the trace containing the key used by the sampler to decide - # the sample rate. This can be helpful in understanding why the sampler is - # making certain decisions about sample rate and help you understand how to - # better choose the sample rate key (aka the FieldList setting above) to use. - AddSampleRateKeyToTrace = true - - # AddSampleRateKeyToTraceField is the name of the field the sampler will use - # when adding the sample rate key to the trace. This setting is only used when - # AddSampleRateKeyToTrace is true. - AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key" - - # ClearFrequencySec is the name of the field the sampler will use to determine - # the period over which it will calculate the sample rate. This setting defaults - # to 30. - # Eligible for live reload. - ClearFrequencySec = 60 - -[dataset2] - - # EMADynamicSampler is a section of the config for manipulating the Exponential - # Moving Average (EMA) Dynamic Sampler implementation. Like the simple DynamicSampler, - # it attempts to average a given sample rate, weighting rare traffic and frequent - # traffic differently so as to end up with the correct average. - # - # EMADynamicSampler is an improvement upon the simple DynamicSampler and is recommended - # for most use cases. Based on the DynamicSampler implementation, EMADynamicSampler differs - # in that rather than compute rate based on a periodic sample of traffic, it maintains an Exponential - # Moving Average of counts seen per key, and adjusts this average at regular intervals. - # The weight applied to more recent intervals is defined by `weight`, a number between - # (0, 1) - larger values weight the average more toward recent observations. In other words, - # a larger weight will cause sample rates more quickly adapt to traffic patterns, - # while a smaller weight will result in sample rates that are less sensitive to bursts or drops - # in traffic and thus more consistent over time. - # - # Keys that are not found in the EMA will always have a sample - # rate of 1. Keys that occur more frequently will be sampled on a logarithmic - # curve. In other words, every key will be represented at least once in any - # given window and more frequent keys will have their sample rate - # increased proportionally to wind up with the goal sample rate. - Sampler = "EMADynamicSampler" - - # GoalSampleRate is the goal rate at which to sample. It indicates a ratio, where - # one sample trace is kept for every n traces seen. For example, a SampleRate of - # 30 will keep 1 out of every 30 traces. This rate is handed to the dynamic - # sampler, who assigns a sample rate for each trace based on the fields selected - # from that trace. - # Eligible for live reload. - GoalSampleRate = 2 - - # FieldList is a list of all the field names to use to form the key that will be - # handed to the dynamic sampler. The cardinality of the combination of values - # from all of these keys should be reasonable in the face of the frequency of - # those keys. If the combination of fields in these keys essentially makes them - # unique, the dynamic sampler will do no sampling. If the keys have too few - # values, you won't get samples of the most interesting traces. A good key - # selection will have consistent values for high frequency boring traffic and - # unique values for outliers and interesting traffic. Including an error field - # (or something like HTTP status code) is an excellent choice. As an example, - # assuming 30 or so endpoints, a combination of HTTP endpoint and status code - # would be a good set of keys in order to let you see accurately use of all - # endpoints and call out when there is failing traffic to any endpoint. Field - # names may come from any span in the trace. - # Eligible for live reload. - FieldList = ["request.method","response.status_code"] - - # UseTraceLength will add the number of spans in the trace in to the dynamic - # sampler as part of the key. The number of spans is exact, so if there are - # normally small variations in trace length you may want to leave this off. If - # traces are consistent lengths and changes in trace length is a useful - # indicator of traces you'd like to see in Honeycomb, set this to true. - # Eligible for live reload. - UseTraceLength = true - - # AddSampleRateKeyToTrace when this is set to true, the sampler will add a field - # to the root span of the trace containing the key used by the sampler to decide - # the sample rate. This can be helpful in understanding why the sampler is - # making certain decisions about sample rate and help you understand how to - # better choose the sample rate key (aka the FieldList setting above) to use. - AddSampleRateKeyToTrace = true - - # AddSampleRateKeyToTraceField is the name of the field the sampler will use - # when adding the sample rate key to the trace. This setting is only used when - # AddSampleRateKeyToTrace is true. - AddSampleRateKeyToTraceField = "meta.refinery.dynsampler_key" - - # AdjustmentInterval defines how often (in seconds) we adjust the moving average from - # recent observations. Default 15s - # Eligible for live reload. - AdjustmentInterval = 15 - - # Weight is a value between (0, 1) indicating the weighting factor used to adjust - # the EMA. With larger values, newer data will influence the average more, and older - # values will be factored out more quickly. In mathematical literature concerning EMA, - # this is referred to as the `alpha` constant. - # Default is 0.5 - # Eligible for live reload. - Weight = 0.5 - - # MaxKeys, if greater than 0, limits the number of distinct keys tracked in EMA. - # Once MaxKeys is reached, new keys will not be included in the sample rate map, but - # existing keys will continue to be be counted. You can use this to keep the sample rate - # map size under control. - # Eligible for live reload - MaxKeys = 0 - - # AgeOutValue indicates the threshold for removing keys from the EMA. The EMA of any key - # will approach 0 if it is not repeatedly observed, but will never truly reach it, so we have to - # decide what constitutes "zero". Keys with averages below this threshold will be removed - # from the EMA. Default is the same as Weight, as this prevents a key with the smallest - # integer value (1) from being aged out immediately. This value should generally be <= Weight, - # unless you have very specific reasons to set it higher. - # Eligible for live reload - AgeOutValue = 0.5 - - # BurstMultiple, if set, is multiplied by the sum of the running average of counts to define - # the burst detection threshold. If total counts observed for a given interval exceed the threshold - # EMA is updated immediately, rather than waiting on the AdjustmentInterval. - # Defaults to 2; negative value disables. With a default of 2, if your traffic suddenly doubles, - # burst detection will kick in. - # Eligible for live reload - BurstMultiple = 2.0 - - # BurstDetectionDelay indicates the number of intervals to run after Start is called before - # burst detection kicks in. - # Defaults to 3 - # Eligible for live reload - BurstDetectionDelay = 3 - -[dataset3] - - Sampler = "DeterministicSampler" - SampleRate = 10 - -[dataset4] - - Sampler = "RulesBasedSampler" - - [[dataset4.rule]] - name = "500 errors" - SampleRate = 1 - [[dataset4.rule.condition]] - field = "status_code" - operator = "=" - value = 500 - [[dataset4.rule.condition]] - field = "duration_ms" - operator = ">=" - value = 1000.789 - - [[dataset4.rule]] - name = "drop 200 responses" - drop = true - [[dataset4.rule.condition]] - field = "status_code" - operator = "=" - value = 200 - - [[dataset4.rule]] - SampleRate = 10 # default when no rules match, if missing defaults to 1 diff --git a/refinery/templates/configmap-config.yaml b/refinery/templates/configmap-config.yaml index 8ee316a2..49906098 100644 --- a/refinery/templates/configmap-config.yaml +++ b/refinery/templates/configmap-config.yaml @@ -5,5 +5,5 @@ metadata: labels: {{- include "refinery.labels" . | nindent 4 }} data: - config.toml: | -{{ tpl (.Files.Get "config.toml") . | indent 6 }} + config.yaml: | +{{- toYaml .Values.config | nindent 4 }} diff --git a/refinery/templates/configmap-rules.yaml b/refinery/templates/configmap-rules.yaml index e18fd1af..42a1f102 100644 --- a/refinery/templates/configmap-rules.yaml +++ b/refinery/templates/configmap-rules.yaml @@ -5,5 +5,5 @@ metadata: labels: {{- include "refinery.labels" . | nindent 4 }} data: - rules.toml: |- -{{ tpl (.Files.Get "rules.toml") . | indent 6 }} + rules.yaml: | +{{- toYaml .Values.rules | nindent 4 }} \ No newline at end of file diff --git a/refinery/templates/deployment.yaml b/refinery/templates/deployment.yaml index 19c8701d..e42afeef 100644 --- a/refinery/templates/deployment.yaml +++ b/refinery/templates/deployment.yaml @@ -36,7 +36,12 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["refinery"] + command: + - "refinery" + - "-c" + - "/etc/refinery/config.yaml" + - "-r" + - "/etc/refinery/rules.yaml" ports: - name: data containerPort: 8080 @@ -65,13 +70,13 @@ spec: - configMap: name: {{ include "refinery.fullname" . }}-config items: - - key: config.toml - path: refinery.toml + - key: config.yaml + path: config.yaml - configMap: name: {{ include "refinery.fullname" . }}-rules items: - - key: rules.toml - path: rules.toml + - key: rules.yaml + path: rules.yaml {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/refinery/values.yaml b/refinery/values.yaml index 57a48e5b..d575fdae 100644 --- a/refinery/values.yaml +++ b/refinery/values.yaml @@ -15,43 +15,49 @@ nameOverride: "" fullnameOverride: "" -# Values used to build config.toml +# Values used to build config.yaml config: - ListenAddr: "0.0.0.0:8080" - PeerListenAddr: "0.0.0.0:8081" - HoneycombAPI: "https://api.honeycomb.io" + ListenAddr: 0.0.0.0:8080 + PeerListenAddr: 0.0.0.0:8081 + HoneycombAPI: https://api.honeycomb.io LoggingLevel: error - SendDelay: "2s" - TraceTimeout: "300s" - SendTicker: "100ms" + SendDelay: 2s + TraceTimeout: 300s + SendTicker: 100ms UpstreamBufferSize: 1000 PeerBufferSize: 1000 - # Implementation Choices - Logger: "logrus" - Metrics: "prometheus" - InMemCollector: CacheCapacity: 1000 MaxAlloc: 0 - HoneycombLogger: - LoggerHoneycombAPI: "https://api.honeycomb.io" - LoggerAPIKey: "YOUR_API_KEY" - LoggerDataset: "refinery-logs" + # Implementation Choices + Logger: logrus + Metrics: prometheus + +# HoneycombLogger: +# LoggerHoneycombAPI: https://api.honeycomb.io +# LoggerAPIKey: YOUR_API_KEY +# LoggerDataset: refinery-logs - HoneycombMetrics: - MetricsHoneycombAPI: "https://api.honeycomb.io" - MetricsAPIKey: "YOUR_API_KEY" - MetricsDataset: "refinery-metrics" - MetricsReportingInterval: 3 +# HoneycombMetrics: +# MetricsHoneycombAPI: https://api.honeycomb.io +# MetricsAPIKey: YOUR_API_KEY +# MetricsDataset: refinery-metrics +# MetricsReportingInterval: 3 -# Values used to configure rules.toml with the same name + PrometheusMetrics: + MetricsListenAddr: 0.0.0.0:9090 + + +# Values used to build rules.yaml rules: - DryRun: false +# DryRun: false Sampler: DeterministicSampler SampleRate: 1 + +# Redis configuration redis: enabled: true # existingHost: From 10459569d8df7a132bc8e3aeb081cf8fa1ca5012 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Sun, 7 Mar 2021 22:17:51 -0500 Subject: [PATCH 03/11] add readme --- refinery/README.md | 158 ++++++++++++++++++++++++++++++++ refinery/templates/service.yaml | 4 + refinery/values.yaml | 7 +- 3 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 refinery/README.md diff --git a/refinery/README.md b/refinery/README.md new file mode 100644 index 00000000..6d7487d8 --- /dev/null +++ b/refinery/README.md @@ -0,0 +1,158 @@ +# Refinery: Sampling Proxy Service for Honeycomb + +[Refinery](https://github.com/honeycombio/refinery) is a trace-aware sampling proxy server for Honeycomb. + +[Honeycomb](https://honeycomb.io) is built for modern software teams to see and understand how their production systems are behaving. +Our goal is to give engineers the observability they need to eliminate toil and delight their users. +This Helm Chart will install Refinery with the desired sampling rules passed in via a configuration file. + +## TL;DR + +```console +helm repo add honeycomb https://honeycombio.github.io/helm-charts +helm install refinery honeycomb/refinery --set rules.SampleRate=2 +``` + +## Prerequisites + +- Helm 3.0+ + +## Installing the Chart + +Add the Honeycomb repo to Helm: + +```console +helm repo add honeycomb https://honeycombio.github.io/helm-charts +``` + +Install Refinery with your custom configuration file: + +```console +helm install refinery honeycomb/refinery --values /path/to/your/refinery.yaml +``` + +If no configuration file is passed in, Refinery will deploy with the default configuration in [`values.yaml`](./values.yaml). + +## Configuring sampling rules + +### Deterministic Sampler + +The default sampling method set in [`values.yaml`](./values.yaml) uses the `DeterministicSampler` with a `SampleRate` of 1. +This means that all of your data is sent to Honeycomb. +Refinery does not down-sample your data by default. + +```yaml +# values.yaml +rules: +# DryRun: false + Sampler: DeterministicSampler + SampleRate: 1 +``` + +To configure custom sampling rules, you will need to pass in your own `refinery.yaml` file. +[The Refinery documentation](https://docs.honeycomb.io/manage-data-volume/refinery/sampling-methods/) goes into more detail about how each sampling method works. +These example configurations are provided to demonstrate how to define rules in YAML. + +### Rules-Based Sampler + +```yaml +# refinery.yaml +rules: +# DryRun: false + my_dataset: + Sampler: "RulesBasedSampler" + rule: + - name: "keep 5xx errors" + SampleRate: 1 + condition: + field: "status_code" + operator: ">=" + value: 500 + - name: "downsample 200 responses" + SampleRate: 1000 + condition: + field: "status_code" + operator: "=" + value: 200 + - name: "send 1 in 10 traces" + SampleRate: 10 # base case +``` + +### Exponential Moving Average (EMA) Dynamic Sampler + +This is the recommended sampling method for most teams. + +```yaml +# refinery.yaml +rules: +# DryRun: false + my_dataset: + Sampler: "EMADynamicSampler" + GoalSampleRate: 2 + FieldList: + - "request.method" + - "response.status_code" + AdjustmentInterval: 15 + Weight: 0.5 +``` + +## Configuration + +The repository's [values.yaml](./values.yaml) file contains information about all configuration options for this chart. + +### Refinery Metrics + +If you decide to send [Refinery's runtime metrics](https://docs.honeycomb.io/manage-data-volume/refinery/scale-and-troubleshoot/#understanding-refinerys-metrics) to Honeycomb, you will need to give Refinery an API for the team you want those metrics sent to. + +You can obtain your API Key by going to your Account profile page inside of your Honeycomb instance. + +```yaml +# refinery.yaml +config: + Metrics: honeycomb + HoneycombMetrics: + # MetricsHoneycombAPI: https://api.honeycomb.io # default + MetricsAPIKey: YOUR_API_KEY + # MetricsDataset: "Refinery Metrics" # default + # MetricsReportingInterval: 3 # default +``` + +## Parameters + +The following table lists the configurable parameters of the Refinery chart and their default values, as defined in [`values.yaml`](./values.yaml). + +| Parameter | Description | Default | +| --- | --- | --- | +| `replicaCount` | Number of Refinery replicas | `2` | +| `image.repository` | Refinery image name | `honeycombio/refinery` | +| `image.pullPolicy` | Refinery image pull policy | `IfNotPresent` | +| `image.tag` | Refinery image tag (leave blank to use app version) | `nil` | +| `imagePullSecrets` | Specify docker-registry secret names as an array | `[]` | +| `nameOverride` | String to partially override refinery.fullname template with a string (will append the release name) | `nil` | +| `fullnameOverride` | String to fully override refinery.fullname template with a string | `nil` | +| `config` | Refinery core Configuration | see [Refinery Configuration](#configuration) | +| `rules` | Refinery sampling rules | see [Configuring sampling rules](#configuring-sampling-rules) | +| `redis.enabled` | When true, a Redis instance will be installed | `true` | +| `redis.existingHost` | If `redis.enabled` is false, this must be set to the name and port of an existing Redis service | `nil` | +| `redis.image.repository` | Redis image name | `redis` | +| `redis.image.tag` | Redis image tag | `6.0.2` | +| `redis.image.pullPolicy` | Redis image pull policy | `IfNotPresent` | +| `prometheus.serviceMonitor.create` | Creates service monitor to collect prom metrics (depends on kube-prom) | `false` | +| `serviceAccount.create` | Specify whether a ServiceAccount should be created | `true` | +| `serviceAccount.name` | The name of the ServiceAccount to create | Generated using the `refinery.fullname` template | +| `serviceAccount.annotations` | Annotations to be applied to ServiceAccount | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podSecurityContext` | Security context for pod | `{}` | +| `securityContext` | Security context for container | `{}` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Service port | `80` | +| `service.annotations` | Service annotations | `{}` | +| `ingress.enabled` | Enable ingress controller resource | `false` | +| `ingress.annotations` | Ingress annotations | `{}` | +| `ingress.hosts[0].name` | Hostname to your Refinery installation | `refinery.local` | +| `ingress.hosts[0].paths` | Path within the url structure | `[]` | +| `ingress.tls` | TLS hosts | `[]` | +| `resources` | CPU/Memory resource requests/limits | limit: 1000m/2Gi, request: 500m/500Mi | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Tolerations for pod assignment | `[]`| +| `affinity` | Map of node/pod affinities | `{}` | diff --git a/refinery/templates/service.yaml b/refinery/templates/service.yaml index 8df5bf90..daf0bbc8 100644 --- a/refinery/templates/service.yaml +++ b/refinery/templates/service.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "refinery.fullname" . }} labels: {{- include "refinery.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.service.type }} ports: diff --git a/refinery/values.yaml b/refinery/values.yaml index d575fdae..b8dd1854 100644 --- a/refinery/values.yaml +++ b/refinery/values.yaml @@ -102,6 +102,7 @@ securityContext: {} service: type: ClusterIP port: 80 + annotations: {} ingress: enabled: false @@ -109,12 +110,12 @@ ingress: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: - - host: chart-example.local + - host: refinery.local paths: [] tls: [] - # - secretName: chart-example-tls + # - secretName: refinery-tls # hosts: - # - chart-example.local + # - refinery.local resources: limits: From 32e6ea1062cb17a50b0a956194a5dcbe86112da1 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Mon, 8 Mar 2021 21:44:27 -0500 Subject: [PATCH 04/11] cleanup readme --- refinery/README.md | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/refinery/README.md b/refinery/README.md index 6d7487d8..aaaa31ba 100644 --- a/refinery/README.md +++ b/refinery/README.md @@ -8,6 +8,7 @@ This Helm Chart will install Refinery with the desired sampling rules passed in ## TL;DR +Install the chart with a deterministic sample rate of 1 out of every 2 events. ```console helm repo add honeycomb https://honeycombio.github.io/helm-charts helm install refinery honeycomb/refinery --set rules.SampleRate=2 @@ -19,44 +20,42 @@ helm install refinery honeycomb/refinery --set rules.SampleRate=2 ## Installing the Chart -Add the Honeycomb repo to Helm: +Install Refinery with your custom chart values file ```console -helm repo add honeycomb https://honeycombio.github.io/helm-charts -``` - -Install Refinery with your custom configuration file: - -```console -helm install refinery honeycomb/refinery --values /path/to/your/refinery.yaml +helm install refinery honeycomb/refinery --values /path/to/refinery-values.yaml ``` If no configuration file is passed in, Refinery will deploy with the default configuration in [`values.yaml`](./values.yaml). ## Configuring sampling rules +The **Sample Rate** in Honeycomb is expressed as the denominator for 1 out of X events. +A sample rate of 20 means to keep 1 event from every 20, which is also equivalent to a 5% sampling you may see with other platforms. + +[The Refinery documentation](https://docs.honeycomb.io/manage-data-volume/refinery/sampling-methods/) goes into more detail about how each sampling method works. +These example configurations are provided to demonstrate how to define rules in YAML. + ### Deterministic Sampler -The default sampling method set in [`values.yaml`](./values.yaml) uses the `DeterministicSampler` with a `SampleRate` of 1. +The default sampling method uses the `DeterministicSampler` with a `SampleRate` of 1. This means that all of your data is sent to Honeycomb. Refinery does not down-sample your data by default. ```yaml -# values.yaml rules: # DryRun: false Sampler: DeterministicSampler SampleRate: 1 ``` -To configure custom sampling rules, you will need to pass in your own `refinery.yaml` file. -[The Refinery documentation](https://docs.honeycomb.io/manage-data-volume/refinery/sampling-methods/) goes into more detail about how each sampling method works. -These example configurations are provided to demonstrate how to define rules in YAML. +### Rules Based Sampler -### Rules-Based Sampler +The Rules Based Sampler allows you to specify specific span attribute values that will be used to determine a sample rate applied to the trace. +Rules are evaluated in order as they appear in the configuration. +A default rule should also be specified as the last rule. ```yaml -# refinery.yaml rules: # DryRun: false my_dataset: @@ -80,10 +79,12 @@ rules: ### Exponential Moving Average (EMA) Dynamic Sampler -This is the recommended sampling method for most teams. +The EMA Dynamic Sampler, will determine sampling rates, based on field values. +Values that are seen often (ie: http status 200) will get sampled less than values that are rare (ie: http status 500). +You configure which fields to use for sampling decisions, the exponential moving average adjustment interval, and weight. +This is the recommended sampling method for most teams. ```yaml -# refinery.yaml rules: # DryRun: false my_dataset: @@ -104,7 +105,7 @@ The repository's [values.yaml](./values.yaml) file contains information about al If you decide to send [Refinery's runtime metrics](https://docs.honeycomb.io/manage-data-volume/refinery/scale-and-troubleshoot/#understanding-refinerys-metrics) to Honeycomb, you will need to give Refinery an API for the team you want those metrics sent to. -You can obtain your API Key by going to your Account profile page inside of your Honeycomb instance. +You can obtain your API Key by going to your Account profile page within your Honeycomb instance. ```yaml # refinery.yaml From 8605ae3ee5b2fe219a9dc9e3a5cf17972555e299 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Tue, 9 Mar 2021 20:56:46 -0500 Subject: [PATCH 05/11] fix rulesbasedsampler example in readme --- refinery/Chart.yaml | 2 +- refinery/README.md | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/refinery/Chart.yaml b/refinery/Chart.yaml index 36a8ff04..f63cbf87 100644 --- a/refinery/Chart.yaml +++ b/refinery/Chart.yaml @@ -3,7 +3,7 @@ name: refinery description: Chart to deploy Honeycomb Refinery type: application version: 0.9.0 -appVersion: 0.16.0 +appVersion: 1.0.1 keywords: - refinery - honeycomb diff --git a/refinery/README.md b/refinery/README.md index aaaa31ba..2a6ab235 100644 --- a/refinery/README.md +++ b/refinery/README.md @@ -60,20 +60,20 @@ rules: # DryRun: false my_dataset: Sampler: "RulesBasedSampler" - rule: - - name: "keep 5xx errors" + Rule: + - Name: "keep 5xx errors" SampleRate: 1 - condition: - field: "status_code" - operator: ">=" - value: 500 - - name: "downsample 200 responses" + Condition: + - Field: "status_code" + Operator: ">=" + Value: 500 + - Name: "downsample 200 responses" SampleRate: 1000 - condition: - field: "status_code" - operator: "=" - value: 200 - - name: "send 1 in 10 traces" + Condition: + - Field: "http.status_code" + Operator: "=" + Value: 200 + - Name: "send 1 in 10 traces" SampleRate: 10 # base case ``` From 3d8c6dea8646a9f9372eb2f7be861800169f5252 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Tue, 9 Mar 2021 20:58:41 -0500 Subject: [PATCH 06/11] add refinery to root readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 055f6bd8..5d899a36 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ You can use Helm for installing [Honeycomb](https://honeycomb.io) packages in yo Packages: - [Honeycomb Kubernetes Agent](./honeycomb/) +- [Honeycomb Refinery](./refinery) - [Honeycomb Secure Tenancy Proxy](./secure-tenancy) - [OpenTelemetry-Collector](./opentelemetry-collector) From 06ea51b252062d9c10e0be78856bcc2b07512ce3 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Wed, 17 Mar 2021 17:52:07 -0400 Subject: [PATCH 07/11] remove prom servicemonitor --- refinery/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/refinery/README.md b/refinery/README.md index 2a6ab235..911397e1 100644 --- a/refinery/README.md +++ b/refinery/README.md @@ -138,7 +138,6 @@ The following table lists the configurable parameters of the Refinery chart and | `redis.image.repository` | Redis image name | `redis` | | `redis.image.tag` | Redis image tag | `6.0.2` | | `redis.image.pullPolicy` | Redis image pull policy | `IfNotPresent` | -| `prometheus.serviceMonitor.create` | Creates service monitor to collect prom metrics (depends on kube-prom) | `false` | | `serviceAccount.create` | Specify whether a ServiceAccount should be created | `true` | | `serviceAccount.name` | The name of the ServiceAccount to create | Generated using the `refinery.fullname` template | | `serviceAccount.annotations` | Annotations to be applied to ServiceAccount | `{}` | From b2e9517667bb96cdd723f243c78d6abab53ba5a3 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Wed, 17 Mar 2021 17:52:26 -0400 Subject: [PATCH 08/11] add environment variable support --- refinery/templates/deployment.yaml | 4 ++++ refinery/values.yaml | 27 +++++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/refinery/templates/deployment.yaml b/refinery/templates/deployment.yaml index e42afeef..c89eecc9 100644 --- a/refinery/templates/deployment.yaml +++ b/refinery/templates/deployment.yaml @@ -42,6 +42,10 @@ spec: - "/etc/refinery/config.yaml" - "-r" - "/etc/refinery/rules.yaml" + {{- with .Values.environment }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} ports: - name: data containerPort: 8080 diff --git a/refinery/values.yaml b/refinery/values.yaml index b8dd1854..589bc5b0 100644 --- a/refinery/values.yaml +++ b/refinery/values.yaml @@ -1,6 +1,4 @@ # Default values for refinery. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. replicaCount: 2 @@ -10,10 +8,21 @@ image: # Overrides the image tag whose default is the chart appVersion. tag: "" -imagePullSecrets: [] +imagePullSecrets: [ ] nameOverride: "" fullnameOverride: "" +# Use to pass in additional environment variables into Refinery +# Refinery supports environment variables for some configuration options such as: +# - Honeycomb API Key used by Logger and Metrics: REFINERY_HONEYCOMB_API_KEY +# - Redis Host: REFINERY_REDIS_HOST +# - Redis Password: REFINERY_REDIS_PASSWORD +environment: [ ] + # - name: REFINERY_HONEYCOMB_API_KEY + # valueFrom: + # secretKeyRef: + # name: honeycomb +# key: api-key # Values used to build config.yaml config: @@ -31,10 +40,10 @@ config: CacheCapacity: 1000 MaxAlloc: 0 - # Implementation Choices Logger: logrus Metrics: prometheus + # If using the Honeycomb Logger or Metrics, you can specify the API Key using an environment variable instead. # HoneycombLogger: # LoggerHoneycombAPI: https://api.honeycomb.io # LoggerAPIKey: YOUR_API_KEY @@ -67,16 +76,6 @@ redis: pullPolicy: IfNotPresent -# Creates service monitor to collect prom metrics (depends on kube-prom) -# prometheus: -# serviceMonitor: -# create: true -# namespace: monitoring -prometheus: - serviceMonitor: - create: false - - serviceAccount: # Specifies whether a service account should be created create: true From 2230d5e79e7c9733cb625fe953df6450c9701ed4 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Wed, 24 Mar 2021 17:21:41 -0400 Subject: [PATCH 09/11] update chart and app versions --- refinery/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/Chart.yaml b/refinery/Chart.yaml index f63cbf87..ec4bb8fe 100644 --- a/refinery/Chart.yaml +++ b/refinery/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: refinery description: Chart to deploy Honeycomb Refinery type: application -version: 0.9.0 -appVersion: 1.0.1 +version: 1.0.0 +appVersion: 1.1.0 keywords: - refinery - honeycomb From cfcb3676cd72394710e7f8c054ffffba78966579 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Thu, 25 Mar 2021 22:31:23 -0400 Subject: [PATCH 10/11] update refinery version to 1.1.1 --- refinery/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refinery/Chart.yaml b/refinery/Chart.yaml index ec4bb8fe..bd1b656e 100644 --- a/refinery/Chart.yaml +++ b/refinery/Chart.yaml @@ -3,7 +3,7 @@ name: refinery description: Chart to deploy Honeycomb Refinery type: application version: 1.0.0 -appVersion: 1.1.0 +appVersion: 1.1.1 keywords: - refinery - honeycomb From 6c0078dc7de2516e2ce1e7eedbe0ebad54d6975f Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Thu, 25 Mar 2021 23:10:47 -0400 Subject: [PATCH 11/11] fix liveness probe --- refinery/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refinery/templates/deployment.yaml b/refinery/templates/deployment.yaml index c89eecc9..c5cef067 100644 --- a/refinery/templates/deployment.yaml +++ b/refinery/templates/deployment.yaml @@ -64,7 +64,7 @@ spec: livenessProbe: httpGet: path: /alive - port: incoming + port: data resources: {{- toYaml .Values.resources | nindent 12 }} volumes: