From c5deb61da23e0d5bf8b0d5f49793737a34571a86 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:25:57 +0000 Subject: [PATCH 1/5] feat: add syslog receiver --- internal/collector/factories.go | 2 ++ internal/collector/factories_test.go | 2 +- .../collector/otel_collector_plugin_test.go | 10 +++++++++ internal/collector/otelcol.tmpl | 22 +++++++++++++++++++ internal/config/types.go | 7 ++++++ .../test-opentelemetry-collector-agent.yaml | 12 ++++++++++ test/docker/nginx-plus/deb/Dockerfile | 19 +++++++++------- test/types/config.go | 5 +++++ 8 files changed, 70 insertions(+), 9 deletions(-) diff --git a/internal/collector/factories.go b/internal/collector/factories.go index df3a431f6..75e6e5019 100644 --- a/internal/collector/factories.go +++ b/internal/collector/factories.go @@ -32,6 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -110,6 +111,7 @@ func createReceiverFactories() (map[component.Type]receiver.Factory, error) { hostmetricsreceiver.NewFactory(), nginxreceiver.NewFactory(), nginxplusreceiver.NewFactory(), + syslogreceiver.NewFactory(), } return receiver.MakeFactoryMap(receiverList...) diff --git a/internal/collector/factories_test.go b/internal/collector/factories_test.go index 5cefda51e..ca3d9423d 100644 --- a/internal/collector/factories_test.go +++ b/internal/collector/factories_test.go @@ -17,7 +17,7 @@ func TestOTelComponentFactories(t *testing.T) { require.NoError(t, err, "OTelComponentFactories should not return an error") assert.NotNil(t, factories, "factories should not be nil") - assert.Len(t, factories.Receivers, 4) + assert.Len(t, factories.Receivers, 5) assert.Len(t, factories.Processors, 20) assert.Len(t, factories.Exporters, 4) assert.Len(t, factories.Extensions, 3) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 9e14c73cf..99ae3c305 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,6 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ { @@ -157,6 +162,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index e6b3baca8..b49e5f9f6 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,4 +1,10 @@ receivers: +{{- if ne .Receivers.SyslogReceiver nil }} + syslog: + tcp: + listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" + protocol: {{ .Receivers.SyslogReceiver.Protocol }} +{{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: collection_interval: {{ .Receivers.HostMetrics.CollectionInterval }} @@ -233,3 +239,19 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + logs: + receivers: + {{- if ne .Receivers.SyslogReceiver nil }} + - syslog + {{- end }} + processors: + {{- if ne .Processors.Batch nil }} + - batch + {{- end }} + exporters: + {{- range $index, $otlpExporter := .Exporters.OtlpExporters }} + - otlp/{{$index}} + {{- end }} + {{- if ne .Exporters.Debug nil }} + - debug + {{- end }} diff --git a/internal/config/types.go b/internal/config/types.go index 0e36ae081..2ec27c880 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,6 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` + SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -171,6 +172,12 @@ type ( OtlpTLSConfig *OtlpTLSConfig `yaml:"-" mapstructure:"tls"` } + SyslogReceiver struct { + Host string `yaml:"-" mapstructure:"host"` + Port string `yaml:"-" mapstructure:"port"` + Protocol string `yaml:"-" mapstructure:"protocol"` + } + NginxReceiver struct { InstanceID string `yaml:"-" mapstructure:"instance_id"` StubStatus string `yaml:"-" mapstructure:"stub_status"` diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index a4f5084a2..c88d3929e 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,4 +1,8 @@ receivers: + syslog: + tcp: + listen_address: "127.0.0.1:515" + protocol: rfc5424 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -84,3 +88,11 @@ service: - otlp/0 - prometheus - debug + logs: + receivers: + - syslog + processors: + - batch + exporters: + - otlp/0 + - debug diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 0754e4bd0..6fa083c6f 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -14,7 +14,7 @@ COPY $ENTRY_POINT /agent/entrypoint.sh RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ -# Create nginx user/group first, to be consistent throughout Docker variants + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -37,20 +37,23 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ done; \ test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# Install the latest release of NGINX Plus and/or NGINX Plus modules -# Uncomment individual modules if necessary -# Use versioned packages over defaults to specify a release - && nginxPackages=" \ - nginx-plus \ - " \ + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Uncomment individual modules if necessary + # Use versioned packages over defaults to specify a release + && nginxPackages="nginx-plus app-protect" \ + # Enable certificate-based authentication for the NGINX repository && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Set up repository sources for NGINX Plus and App Protect && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + # Copy certificates and keys for repository access && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + # Update and install NGINX packages && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ $nginxPackages \ @@ -59,7 +62,7 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ jq \ gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/types/config.go b/test/types/config.go index 3d570c915..ecc97c860 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,6 +67,11 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, InitialDelay: time.Second, From 016275714d8d11f0dc54b70bbe11e4157c776248 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:55:41 +0000 Subject: [PATCH 2/5] chore: update port to non priviledged --- internal/collector/otel_collector_plugin_test.go | 4 ++-- test/config/collector/test-opentelemetry-collector-agent.yaml | 2 +- test/types/config.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 99ae3c305..608a7a73f 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -122,7 +122,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, OtlpReceivers: types.OtlpReceivers(), @@ -164,7 +164,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { OtlpReceivers: types.OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, NginxReceivers: []config.NginxReceiver{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index c88d3929e..29bd572f5 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,7 +1,7 @@ receivers: syslog: tcp: - listen_address: "127.0.0.1:515" + listen_address: "127.0.0.1:1515" protocol: rfc5424 hostmetrics: collection_interval: 1m0s diff --git a/test/types/config.go b/test/types/config.go index ecc97c860..bd88cfc19 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -69,7 +69,7 @@ func AgentConfig() *config.Config { OtlpReceivers: OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, HostMetrics: &config.HostMetrics{ From 036aba93c7bfc2786dd754eab5703452bbf65ad8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:00:04 +0000 Subject: [PATCH 3/5] chore: update the syslog type and template --- internal/collector/otel_collector_plugin.go | 11 +++ .../collector/otel_collector_plugin_test.go | 18 ++-- internal/collector/otelcol.tmpl | 12 +-- internal/config/defaults.go | 1 + internal/config/types.go | 8 +- internal/model/config.go | 7 ++ .../watcher/instance/nginx_config_parser.go | 17 ++++ .../test-opentelemetry-collector-agent.yaml | 6 +- test/docker/nginx-plus/deb/Dockerfile | 96 ++++++++++--------- test/types/config.go | 10 +- 10 files changed, 115 insertions(+), 71 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index d2879dbfb..822071d8c 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -365,6 +365,16 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo PlusAPI: nginxConfigContext.PlusAPI, }, ) + if nginxConfigContext.Syslog != nil && nginxConfigContext.Syslog.SyslogServer != "" { + oc.config.Collector.Receivers.SyslogReceivers = append( + oc.config.Collector.Receivers.SyslogReceivers, + config.SyslogReceiver{ + InstanceID: nginxConfigContext.InstanceID, + Server: nginxConfigContext.Syslog.SyslogServer, + Protocol: "rfc3164", // default value, need to get from the agent conf + }, + ) + } reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { @@ -387,6 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } +// need to update new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 608a7a73f..5727e764d 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,10 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ @@ -162,10 +163,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, NginxReceivers: []config.NginxReceiver{ { diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index b49e5f9f6..8d5de0fdf 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,9 +1,9 @@ receivers: -{{- if ne .Receivers.SyslogReceiver nil }} - syslog: +{{- range .Receivers.SyslogReceivers }} + syslog/{{- .InstanceID -}}: tcp: - listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" - protocol: {{ .Receivers.SyslogReceiver.Protocol }} + listen_address: "{{- .Server -}}" + protocol: rfc3164 {{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: @@ -241,8 +241,8 @@ service: {{- end }} logs: receivers: - {{- if ne .Receivers.SyslogReceiver nil }} - - syslog + {{- range .Receivers.SyslogReceivers }} + - syslog/{{- .InstanceID -}} {{- end }} processors: {{- if ne .Processors.Batch nil }} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 526ea914c..b5de7ffcb 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,6 +12,7 @@ import ( ) const ( + // Add default nap parameter DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/config/types.go b/internal/config/types.go index 2ec27c880..996c18816 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,7 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` - SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` + SyslogReceivers []SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -173,9 +173,9 @@ type ( } SyslogReceiver struct { - Host string `yaml:"-" mapstructure:"host"` - Port string `yaml:"-" mapstructure:"port"` - Protocol string `yaml:"-" mapstructure:"protocol"` + InstanceID string `yaml:"-" mapstructure:"instance_id"` + Server string `yaml:"-" mapstructure:"host"` + Protocol string `yaml:"-" mapstructure:"protocol"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index 3f3742891..ef60d0e56 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -18,6 +18,7 @@ type NginxConfigContext struct { Files []*v1.File AccessLogs []*AccessLog ErrorLogs []*ErrorLog + Syslog *NAP } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { @@ -85,6 +86,12 @@ type ErrorLog struct { Readable bool } +type NAP struct { + Enable bool + Syslog bool + SyslogServer string +} + type ( WriteStatus int ) diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index ab147e2bd..e0f7621e1 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -36,6 +37,7 @@ const ( stubStatusAPIDirective = "stub_status" apiFormat = "http://%s%s" locationDirective = "location" + napDirective = "app_protect_security_log" ) type ( @@ -104,6 +106,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "log_format": formatMap = ncp.formatMap(directive) case "access_log": + fmt.Println("am here") if !ncp.ignoreLog(directive.Args[0]) { accessLog := ncp.accessLog(directive.Args[0], ncp.accessLogDirectiveFormat(directive), formatMap) @@ -120,6 +123,20 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "ssl_certificate", "proxy_ssl_certificate", "ssl_client_certificate", "ssl_trusted_certificate": sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) + case "app_protect_security_log": + // Extract the syslog:server part of the app_protect_security_log directive + if len(directive.Args) > 1 { + syslogArg := directive.Args[1] + re := regexp.MustCompile(`syslog:server=([\S]+)`) + matches := re.FindStringSubmatch(syslogArg) + if len(matches) > 1 { + syslogServer := matches[1] + nginxConfigContext.Syslog.SyslogServer = syslogServer + slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) + fmt.Println("syslogServer") + fmt.Println(syslogServer) + } + } } return nil diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 29bd572f5..8d6be2900 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,8 +1,8 @@ receivers: - syslog: + syslog/1: tcp: listen_address: "127.0.0.1:1515" - protocol: rfc5424 + protocol: rfc3164 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -90,7 +90,7 @@ service: - debug logs: receivers: - - syslog + - syslog/1 processors: - batch exporters: diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 6fa083c6f..9951d5114 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,79 +1,83 @@ +# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " +# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO +# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh +# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -x \ - # Create nginx user/group first, to be consistent throughout Docker variants + set -eux \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ + \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg1 \ - lsb-release \ - git \ - wget \ - make \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ - # Install the latest release of NGINX Plus and/or NGINX Plus modules - # Uncomment individual modules if necessary - # Use versioned packages over defaults to specify a release - && nginxPackages="nginx-plus app-protect" \ - # Enable certificate-based authentication for the NGINX repository - && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ - # Set up repository sources for NGINX Plus and App Protect - && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - # Copy certificates and keys for repository access + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ + \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - # Update and install NGINX packages + \ + # Add NGINX repositories securely + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ + \ + && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + \ && apt-get update \ + && apt-cache show nginx-plus || echo "nginx-plus package not found" \ + && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - $nginxPackages \ - curl \ - gettext-base \ - jq \ - gnupg2 \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ + \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ - && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y \ + \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/nginx-plus.list \ + /etc/apt/sources.list.d/nginx-app-protect.list \ + /etc/apt/apt.conf.d/90nginx \ + /etc/ssl/nginx -EXPOSE 80 +# Expose ports +EXPOSE 80 443 +# Set stop signal STOPSIGNAL SIGQUIT +# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb - -STOPSIGNAL SIGTERM -EXPOSE 80 443 +# Install NGINX Agent package +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } +# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/test/types/config.go b/test/types/config.go index bd88cfc19..d1e5cfccf 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,10 +67,12 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164", + }, }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, From 0e138e7ccb330b248603591c947b5486c4bae498 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:14:28 +0000 Subject: [PATCH 4/5] chore: update comments --- internal/collector/otel_collector_plugin.go | 2 +- internal/config/defaults.go | 2 +- .../watcher/instance/nginx_config_parser.go | 1 - test/docker/nginx-plus/deb/Dockerfile | 67 ++++++------------- 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 822071d8c..f0d8c813c 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -397,7 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } -// need to update new syslogreceivers +// Todo: consider update scenario for new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index b5de7ffcb..b386a236b 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,7 +12,7 @@ import ( ) const ( - // Add default nap parameter + //Todo: Add default Protocol for nap DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index e0f7621e1..52f0c09ee 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -124,7 +124,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) case "app_protect_security_log": - // Extract the syslog:server part of the app_protect_security_log directive if len(directive.Args) > 1 { syslogArg := directive.Args[1] re := regexp.MustCompile(`syslog:server=([\S]+)`) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 9951d5114..1cd29f0b3 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,15 +1,12 @@ -# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " -# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO -# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh @@ -17,67 +14,45 @@ COPY $ENTRY_POINT /agent/entrypoint.sh # Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -eux \ + set -x \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ - \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg2 \ - lsb-release \ - git \ - wget \ - make \ - apt-transport-https \ - ubuntu-keyring \ - \ + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - \ - # Add NGINX repositories securely && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ - \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ - \ && apt-get update \ - && apt-cache show nginx-plus || echo "nginx-plus package not found" \ - && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - nginx-plus \ - app-protect \ - app-protect-attack-signatures \ - curl \ - gettext-base \ - jq \ - \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y \ - \ - # Clean up - && rm -rf /var/lib/apt/lists/* \ - /etc/apt/sources.list.d/nginx-plus.list \ - /etc/apt/sources.list.d/nginx-app-protect.list \ - /etc/apt/apt.conf.d/90nginx \ - /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -# Expose ports EXPOSE 80 443 -# Set stop signal STOPSIGNAL SIGQUIT -# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb -# Install NGINX Agent package -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } - -# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] From 4f6db4ae8b55624d629377e89135ffafccfa3ce8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 15:49:36 +0000 Subject: [PATCH 5/5] chore: update the docker --- test/docker/nginx-plus/deb/Dockerfile | 39 ++++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 1cd29f0b3..9b1aa0404 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -11,10 +11,10 @@ WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh -# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -25,34 +25,47 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ git \ wget \ make \ - apt-transport-https \ - ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ - && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ - && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Add GPG keys for nginx-plus and app-protect + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ + #&& ls -l /usr/share/keyrings/nginx-archive-keyring.gpg /usr/share/keyrings/app-protect-security-updates.gpg \ + + + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Set up repository sources for NGINX Plus and App Protect && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ + # && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + # && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ nginx-plus \ app-protect \ - app-protect-attack-signatures \ curl \ gettext-base \ jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y \ + && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -EXPOSE 80 443 +EXPOSE 80 STOPSIGNAL SIGQUIT RUN chmod +x /agent/entrypoint.sh RUN apt install -y /agent/build/${PACKAGE_NAME}.deb +STOPSIGNAL SIGTERM + +EXPOSE 80 443 + ENTRYPOINT ["/agent/entrypoint.sh"]