From 64bda60aab126770813e832de172a26f43ba8299 Mon Sep 17 00:00:00 2001 From: Rajat Chopra Date: Thu, 20 Apr 2017 13:13:04 -0400 Subject: [PATCH 1/2] nginx router based on template --- images/router/nginx/.cccp.yml | 1 + images/router/nginx/Dockerfile | 31 ++++ images/router/nginx/conf/default_pub_keys.pem | 34 +++++ images/router/nginx/conf/error-page-503.http | 140 ++++++++++++++++++ .../router/nginx/conf/nginx-config.template | 60 ++++++++ images/router/nginx/reload-nginx | 16 ++ 6 files changed, 282 insertions(+) create mode 100644 images/router/nginx/.cccp.yml create mode 100644 images/router/nginx/Dockerfile create mode 100644 images/router/nginx/conf/default_pub_keys.pem create mode 100644 images/router/nginx/conf/error-page-503.http create mode 100644 images/router/nginx/conf/nginx-config.template create mode 100755 images/router/nginx/reload-nginx diff --git a/images/router/nginx/.cccp.yml b/images/router/nginx/.cccp.yml new file mode 100644 index 000000000000..04812a559fdf --- /dev/null +++ b/images/router/nginx/.cccp.yml @@ -0,0 +1 @@ +job-id: origin-nginx-router diff --git a/images/router/nginx/Dockerfile b/images/router/nginx/Dockerfile new file mode 100644 index 000000000000..5aebfb560ce5 --- /dev/null +++ b/images/router/nginx/Dockerfile @@ -0,0 +1,31 @@ +# +# This is the NGINX router for OpenShift Origin. +# +# The standard name for this image is openshift/origin-nginx-router +# +FROM openshift/origin + +RUN INSTALL_PKGS="nginx" && \ + yum install -y "epel-release" && \ + yum install -y $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + yum clean all && \ + mkdir -p /var/lib/nginx/router/{certs,cacerts} && \ + mkdir -p /var/lib/nginx/{conf,run,bin,log,logs} && \ + touch /var/lib/nginx/conf/{{os_http_be,os_edge_http_be,os_tcp_be,os_sni_passthrough,os_reencrypt,os_route_http_expose,os_route_http_redirect,cert_config,os_wildcard_domain}.map,nginx.config} && \ + setcap 'cap_net_bind_service=ep' /usr/sbin/nginx && \ + chown -R :0 /var/lib/nginx && \ + chown -R :0 /var/log/nginx && \ + chmod -R 777 /var/log/nginx && \ + chmod -R 777 /var/lib/nginx + +COPY . /var/lib/nginx/ + +LABEL io.k8s.display-name="OpenShift Origin NGINX Router" \ + io.k8s.description="This is a component of OpenShift Origin and contains an NGINX instance that automatically exposes services within the cluster through routes, and offers TLS termination, reencryption, or SNI-passthrough on ports 80 and 443." +USER 1001 +EXPOSE 80 443 +WORKDIR /var/lib/nginx/conf +ENV TEMPLATE_FILE=/var/lib/nginx/conf/nginx-config.template \ + RELOAD_SCRIPT=/var/lib/nginx/reload-nginx +ENTRYPOINT ["/usr/bin/openshift-router"] diff --git a/images/router/nginx/conf/default_pub_keys.pem b/images/router/nginx/conf/default_pub_keys.pem new file mode 100644 index 000000000000..fa3783dca9c1 --- /dev/null +++ b/images/router/nginx/conf/default_pub_keys.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIDIjCCAgqgAwIBAgIBBjANBgkqhkiG9w0BAQUFADCBoTELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAlNDMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl +ZmF1bHQgQ29tcGFueSBMdGQxEDAOBgNVBAsMB1Rlc3QgQ0ExGjAYBgNVBAMMEXd3 +dy5leGFtcGxlY2EuY29tMSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUu +Y29tMB4XDTE2MDExMzE5NDA1N1oXDTI2MDExMDE5NDA1N1owfDEYMBYGA1UEAxMP +d3d3LmV4YW1wbGUuY29tMQswCQYDVQQIEwJTQzELMAkGA1UEBhMCVVMxIjAgBgkq +hkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20xEDAOBgNVBAoTB0V4YW1wbGUx +EDAOBgNVBAsTB0V4YW1wbGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM0B +u++oHV1wcphWRbMLUft8fD7nPG95xs7UeLPphFZuShIhhdAQMpvcsFeg+Bg9PWCu +v3jZljmk06MLvuWLfwjYfo9q/V+qOZVfTVHHbaIO5RTXJMC2Nn+ACF0kHBmNcbth +OOgF8L854a/P8tjm1iPR++vHnkex0NH7lyosVc/vAgMBAAGjDTALMAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQEFBQADggEBADjFm5AlNH3DNT1Uzx3m66fFjqqrHEs25geT +yA3rvBuynflEHQO95M/8wCxYVyuAx4Z1i4YDC7tx0vmOn/2GXZHY9MAj1I8KCnwt +Jik7E2r1/yY0MrkawljOAxisXs821kJ+Z/51Ud2t5uhGxS6hJypbGspMS7OtBbw7 +8oThK7cWtCXOldNF6ruqY1agWnhRdAq5qSMnuBXuicOP0Kbtx51a1ugE3SnvQenJ +nZxdtYUXvEsHZC/6bAtTfNh+/SwgxQJuL2ZM+VG3X2JIKY8xTDui+il7uTh422lq +wED8uwKl+bOj6xFDyw4gWoBxRobsbFaME8pkykP1+GnKDberyAM= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDNAbvvqB1dcHKYVkWzC1H7fHw+5zxvecbO1Hiz6YRWbkoSIYXQ +EDKb3LBXoPgYPT1grr942ZY5pNOjC77li38I2H6Pav1fqjmVX01Rx22iDuUU1yTA +tjZ/gAhdJBwZjXG7YTjoBfC/OeGvz/LY5tYj0fvrx55HsdDR+5cqLFXP7wIDAQAB +AoGAfE7P4Zsj6zOzGPI/Izj7Bi5OvGnEeKfzyBiH9Dflue74VRQkqqwXs/DWsNv3 +c+M2Y3iyu5ncgKmUduo5X8D9To2ymPRLGuCdfZTxnBMpIDKSJ0FTwVPkr6cYyyBk +5VCbc470pQPxTAAtl2eaO1sIrzR4PcgwqrSOjwBQQocsGAECQQD8QOra/mZmxPbt +bRh8U5lhgZmirImk5RY3QMPI/1/f4k+fyjkU5FRq/yqSyin75aSAXg8IupAFRgyZ +W7BT6zwBAkEA0A0ugAGorpCbuTa25SsIOMxkEzCiKYvh0O+GfGkzWG4lkSeJqGME +keuJGlXrZNKNoCYLluAKLPmnd72X2yTL7wJARM0kAXUP0wn324w8+HQIyqqBj/gF +Vt9Q7uMQQ3s72CGu3ANZDFS2nbRZFU5koxrggk6lRRk1fOq9NvrmHg10AQJABOea +pgfj+yGLmkUw8JwgGH6xCUbHO+WBUFSlPf+Y50fJeO+OrjqPXAVKeSV3ZCwWjKT4 +9viXJNJJ4WfF0bO/XwJAOMB1wQnEOSZ4v+laMwNtMq6hre5K8woqteXICoGcIWe8 +u3YLAbyW/lHhOCiZu2iAI8AbmXem9lW6Tr7p/97s0w== +-----END RSA PRIVATE KEY----- diff --git a/images/router/nginx/conf/error-page-503.http b/images/router/nginx/conf/error-page-503.http new file mode 100644 index 000000000000..cb826df8673a --- /dev/null +++ b/images/router/nginx/conf/error-page-503.http @@ -0,0 +1,140 @@ +HTTP/1.0 503 Service Unavailable +Pragma: no-cache +Cache-Control: private, max-age=0, no-cache, no-store +Connection: close +Content-Type: text/html + + + + + + + + +
+

Application is not available

+

The application is currently not serving requests at this endpoint. It may not have been started or is still starting.

+ +
+

+ Possible reasons you are seeing this page: +

+
    +
  • + The host doesn't exist. + Make sure the hostname was typed correctly and that a route matching this hostname exists. +
    • +
  • + The host exists, but doesn't have a matching path. + Check if the URL path was typed correctly and that the route was created using the desired path. +
    • +
  • + Route and path matches, but all pods are down. + Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running. +
    • +
+
+
+ + diff --git a/images/router/nginx/conf/nginx-config.template b/images/router/nginx/conf/nginx-config.template new file mode 100644 index 000000000000..84c306a35e55 --- /dev/null +++ b/images/router/nginx/conf/nginx-config.template @@ -0,0 +1,60 @@ +{{/* + nginx.config: contains the main config with helper backends that are used to terminate + encryption before finally sending to a host_be which is the backend that is the final + backend for a route and contains all the endpoints for the service +*/}} +{{- define "/var/lib/nginx/conf/nginx.config" -}} +#user www www; ## Default: nobody +worker_processes 5; ## Default: 1 +error_log /var/lib/nginx/logs/error.log; +pid /var/lib/nginx/logs/nginx.pid; +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; ## Default: 1024 +} + +http { + #include conf/mime.types; + #include /etc/nginx/proxy.conf; + #include /etc/nginx/fastcgi.conf; + index index.html index.htm index.php; + + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/lib/nginx/logs/access.log main; + sendfile on; + tcp_nopush on; + server_names_hash_bucket_size 128; # this seems to be required for some vhosts + + + +{{- range $cfgIdx, $cfg := .State }} + {{- if (eq $cfg.TLSTermination "") }} + +# Plain http backend + upstream be_http_{{$cfgIdx}} { + {{- range $serviceUnitName, $weight := $cfg.ServiceUnitNames }} + {{- with $serviceUnit := index $.ServiceUnits $serviceUnitName }} + {{- range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }} + server {{$endpoint.IP}}:{{$endpoint.Port}}; + {{ end -}} + {{ end -}} + {{ end -}} + } + + server { # simple load balancing + listen 80; + server_name {{$cfg.Host}}; + access_log /var/lib/nginx/logs/be_http_{{$cfgIdx}}.log main; + + location / { + proxy_pass http://be_http_{{$cfgIdx}}; + } + } + {{ end -}}{{/* end if plain http */}} +{{ end -}}{{/* end all routes */}} +} +{{ end -}}{{/* end config file */}} diff --git a/images/router/nginx/reload-nginx b/images/router/nginx/reload-nginx new file mode 100755 index 000000000000..a496804b1db3 --- /dev/null +++ b/images/router/nginx/reload-nginx @@ -0,0 +1,16 @@ +#!/bin/bash + +set -o nounset + +config_file=/var/lib/nginx/conf/nginx.config +old_pids=$(ps -A -opid,args | grep nginx | egrep -v -e 'grep|reload-nginx' | awk '{print $1}' | tr '\n' ' ') + +reload_status=0 +if [ -n "$old_pids" ]; then + /usr/sbin/nginx -c ${config_file} -s reload + reload_status=$? +else + /usr/sbin/nginx -c ${config_file} + reload_status=$? +fi +exit $reload_status From f0bc6b4da60d6c5ac9a3be7ac6cfa1da22e4a598 Mon Sep 17 00:00:00 2001 From: Rajat Chopra Date: Fri, 25 Aug 2017 16:05:11 -0400 Subject: [PATCH 2/2] tls edge support add nginx to build local images script --- hack/build-local-images.py | 9 +++++ images/router/nginx/conf/default_pub_keys.pem | 34 ------------------- .../router/nginx/conf/nginx-config.template | 22 ++++++++---- images/router/nginx/reload-nginx | 5 +-- 4 files changed, 26 insertions(+), 44 deletions(-) delete mode 100644 images/router/nginx/conf/default_pub_keys.pem diff --git a/hack/build-local-images.py b/hack/build-local-images.py index f11cec70fee8..76b3e6e7cdb0 100755 --- a/hack/build-local-images.py +++ b/hack/build-local-images.py @@ -96,6 +96,15 @@ }, "files": {} }, + "nginx-router": { + "directory": "router/nginx", + "binaries": { + "openshift": "/usr/bin/openshift" + }, + "files": { + ".": "/var/lib/nginx" + } + }, "haproxy-router": { "directory": "router/haproxy", "binaries": { diff --git a/images/router/nginx/conf/default_pub_keys.pem b/images/router/nginx/conf/default_pub_keys.pem deleted file mode 100644 index fa3783dca9c1..000000000000 --- a/images/router/nginx/conf/default_pub_keys.pem +++ /dev/null @@ -1,34 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDIjCCAgqgAwIBAgIBBjANBgkqhkiG9w0BAQUFADCBoTELMAkGA1UEBhMCVVMx -CzAJBgNVBAgMAlNDMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl -ZmF1bHQgQ29tcGFueSBMdGQxEDAOBgNVBAsMB1Rlc3QgQ0ExGjAYBgNVBAMMEXd3 -dy5leGFtcGxlY2EuY29tMSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUu -Y29tMB4XDTE2MDExMzE5NDA1N1oXDTI2MDExMDE5NDA1N1owfDEYMBYGA1UEAxMP -d3d3LmV4YW1wbGUuY29tMQswCQYDVQQIEwJTQzELMAkGA1UEBhMCVVMxIjAgBgkq -hkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20xEDAOBgNVBAoTB0V4YW1wbGUx -EDAOBgNVBAsTB0V4YW1wbGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM0B -u++oHV1wcphWRbMLUft8fD7nPG95xs7UeLPphFZuShIhhdAQMpvcsFeg+Bg9PWCu -v3jZljmk06MLvuWLfwjYfo9q/V+qOZVfTVHHbaIO5RTXJMC2Nn+ACF0kHBmNcbth -OOgF8L854a/P8tjm1iPR++vHnkex0NH7lyosVc/vAgMBAAGjDTALMAkGA1UdEwQC -MAAwDQYJKoZIhvcNAQEFBQADggEBADjFm5AlNH3DNT1Uzx3m66fFjqqrHEs25geT -yA3rvBuynflEHQO95M/8wCxYVyuAx4Z1i4YDC7tx0vmOn/2GXZHY9MAj1I8KCnwt -Jik7E2r1/yY0MrkawljOAxisXs821kJ+Z/51Ud2t5uhGxS6hJypbGspMS7OtBbw7 -8oThK7cWtCXOldNF6ruqY1agWnhRdAq5qSMnuBXuicOP0Kbtx51a1ugE3SnvQenJ -nZxdtYUXvEsHZC/6bAtTfNh+/SwgxQJuL2ZM+VG3X2JIKY8xTDui+il7uTh422lq -wED8uwKl+bOj6xFDyw4gWoBxRobsbFaME8pkykP1+GnKDberyAM= ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDNAbvvqB1dcHKYVkWzC1H7fHw+5zxvecbO1Hiz6YRWbkoSIYXQ -EDKb3LBXoPgYPT1grr942ZY5pNOjC77li38I2H6Pav1fqjmVX01Rx22iDuUU1yTA -tjZ/gAhdJBwZjXG7YTjoBfC/OeGvz/LY5tYj0fvrx55HsdDR+5cqLFXP7wIDAQAB -AoGAfE7P4Zsj6zOzGPI/Izj7Bi5OvGnEeKfzyBiH9Dflue74VRQkqqwXs/DWsNv3 -c+M2Y3iyu5ncgKmUduo5X8D9To2ymPRLGuCdfZTxnBMpIDKSJ0FTwVPkr6cYyyBk -5VCbc470pQPxTAAtl2eaO1sIrzR4PcgwqrSOjwBQQocsGAECQQD8QOra/mZmxPbt -bRh8U5lhgZmirImk5RY3QMPI/1/f4k+fyjkU5FRq/yqSyin75aSAXg8IupAFRgyZ -W7BT6zwBAkEA0A0ugAGorpCbuTa25SsIOMxkEzCiKYvh0O+GfGkzWG4lkSeJqGME -keuJGlXrZNKNoCYLluAKLPmnd72X2yTL7wJARM0kAXUP0wn324w8+HQIyqqBj/gF -Vt9Q7uMQQ3s72CGu3ANZDFS2nbRZFU5koxrggk6lRRk1fOq9NvrmHg10AQJABOea -pgfj+yGLmkUw8JwgGH6xCUbHO+WBUFSlPf+Y50fJeO+OrjqPXAVKeSV3ZCwWjKT4 -9viXJNJJ4WfF0bO/XwJAOMB1wQnEOSZ4v+laMwNtMq6hre5K8woqteXICoGcIWe8 -u3YLAbyW/lHhOCiZu2iAI8AbmXem9lW6Tr7p/97s0w== ------END RSA PRIVATE KEY----- diff --git a/images/router/nginx/conf/nginx-config.template b/images/router/nginx/conf/nginx-config.template index 84c306a35e55..449daa1dbf54 100644 --- a/images/router/nginx/conf/nginx-config.template +++ b/images/router/nginx/conf/nginx-config.template @@ -4,6 +4,7 @@ backend for a route and contains all the endpoints for the service */}} {{- define "/var/lib/nginx/conf/nginx.config" -}} +{{- $workingDir := .WorkingDir }} #user www www; ## Default: nobody worker_processes 5; ## Default: 1 error_log /var/lib/nginx/logs/error.log; @@ -32,10 +33,8 @@ http { {{- range $cfgIdx, $cfg := .State }} - {{- if (eq $cfg.TLSTermination "") }} -# Plain http backend - upstream be_http_{{$cfgIdx}} { + upstream be_{{$cfg.Namespace}}_{{$cfg.Name}} { {{- range $serviceUnitName, $weight := $cfg.ServiceUnitNames }} {{- with $serviceUnit := index $.ServiceUnits $serviceUnitName }} {{- range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }} @@ -46,15 +45,26 @@ http { } server { # simple load balancing + {{- if (eq $cfg.TLSTermination "") }} listen 80; + {{- else }} + listen 443 ssl; + {{ end -}} + server_name {{$cfg.Host}}; - access_log /var/lib/nginx/logs/be_http_{{$cfgIdx}}.log main; + {{- if and (ne $cfg.Host "") (or (eq $cfg.TLSTermination "edge") (eq $cfg.TLSTermination "reencrypt")) -}} + {{ $cert := index $cfg.Certificates $cfg.Host -}} + {{ if ne $cert.Contents "" }} + ssl_certificate {{$workingDir}}/certs/{{$cfgIdx}}.pem; + ssl_certificate_key {{$workingDir}}/certs/{{$cfgIdx}}.pem; + {{ end -}} + {{ end -}} + access_log /var/lib/nginx/logs/be_{{$cfgIdx}}.log main; location / { - proxy_pass http://be_http_{{$cfgIdx}}; + proxy_pass http://be_{{$cfg.Namespace}}_{{$cfg.Name}}; } } - {{ end -}}{{/* end if plain http */}} {{ end -}}{{/* end all routes */}} } {{ end -}}{{/* end config file */}} diff --git a/images/router/nginx/reload-nginx b/images/router/nginx/reload-nginx index a496804b1db3..f20f1070b44a 100755 --- a/images/router/nginx/reload-nginx +++ b/images/router/nginx/reload-nginx @@ -3,10 +3,7 @@ set -o nounset config_file=/var/lib/nginx/conf/nginx.config -old_pids=$(ps -A -opid,args | grep nginx | egrep -v -e 'grep|reload-nginx' | awk '{print $1}' | tr '\n' ' ') - -reload_status=0 -if [ -n "$old_pids" ]; then +if [ -f /var/lib/nginx/logs/nginx.pid ]; then /usr/sbin/nginx -c ${config_file} -s reload reload_status=$? else