Merge pull request #1472 from jeremymv2/jeremymv2/habitize_the_things

Adding Habitization - this PR supersedes PR#1416

.gitignore

@@ -29,3 +29,7 @@ compile_commands.json
 *.tfstate
 *.tfstate.backup
 *_override.tf
+
+# Habitat
+results
+

docker-compose.yml

@@ -0,0 +1,128 @@
+# Configurable environment variables:
+# HAB_ORIGIN - denotes the docker origin (dockerhub ID)
+# VERSION -  the version identifier tag on the packages
+# AUTOMATE_ENABLED - enable the Automate data collector (true or false)
+# AUTOMATE_SERVER - the IP address or hostname of the Automate server
+# AUTOMATE_TOKEN - the token for the Automate server data collector
+# USER_ID - the user ID to use
+# GROUP_ID - the group ID to use
+
+version: '2.1'
+services:
+  postgresql:
+    image: chefdemo/postgresql:stable
+    environment:
+      HAB_POSTGRESQL: |
+        [superuser]
+        name = 'hab'
+        password = 'chefrocks'
+    volumes:
+      - postgresql-data:/hab/svc/postgresql/data
+
+  chef-server-ctl:
+    image: ${HAB_ORIGIN:-chefserverofficial}/chef-server-ctl:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+    environment:
+      HAB_CHEF_SERVER_CTL: |
+        [chef_server_api]
+        ip = '${HOST_IP:-172.17.0.1}'
+        [secrets.data_collector]
+        token = "${AUTOMATE_TOKEN:-93a49a4f2482c64126f7b6015e6b0f30284287ee4054ff8807fb63d9cbd1c506}"
+
+  elasticsearch:
+    image: chefdemo/elasticsearch:stable
+    command: --peer postgresql
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - elasticsearch-data:/hab/svc/elasticsearch/data
+
+  oc_id:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_id:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  bookshelf:
+    image: ${HAB_ORIGIN:-chefserverofficial}/bookshelf:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  oc_bifrost:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_bifrost:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  oc_erchef:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_erchef:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind bookshelf:bookshelf.default
+      --bind oc_bifrost:oc_bifrost.default
+      --bind database:postgresql.default
+      --bind elasticsearch:elasticsearch.default
+      --bind chef-server-ctl:chef-server-ctl.default
+    environment:
+      HAB_OC_ERCHEF: |
+        [chef_authn]
+        keygen_cache_workers=2
+        keygen_cache_size=10
+        keygen_start_size=0
+        keygen_timeout=20000
+        [data_collector]
+        enabled = ${AUTOMATE_ENABLED:-false}
+        server = "${AUTOMATE_SERVER:-localhost}"
+        port = 443
+
+  chef-server-nginx:
+    image: ${HAB_ORIGIN:-chefserverofficial}/chef-server-nginx:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind oc_erchef:oc_erchef.default
+      --bind oc_bifrost:oc_bifrost.default
+      --bind oc_id:oc_id.default
+      --bind bookshelf:bookshelf.default
+      --bind elasticsearch:elasticsearch.default
+      --bind chef-server-ctl:chef-server-ctl.default
+    volumes:
+      - nginx-data:/hab/svc/chef-server-nginx/data
+    ports:
+      - 80:8080
+      - 443:8443
+
+volumes:
+  postgresql-data:
+  elasticsearch-data:
+  nginx-data:

habitat_pkgs_build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# this script builds all the essential habitat packages for running Chef Server
+# additionaly, it exports them as a local docker image
+
+CHEF_SERVER_SRC='/src/src'
+
+for dir in dbdpg oc-id openresty-noroot nginx bookshelf chef-server-ctl oc_bifrost oc_erchef; do
+  cd $CHEF_SERVER_SRC/$dir
+  echo "[STATUS] building $dir"
+  build > /var/log/build-${dir}-$(date +%s).log
+  if [[ $dir =~ dbdpg ]]; then continue; fi
+  echo "[STATUS] exporting $dir pkg to docker daemon"
+  hab pkg export docker -i "chefserverofficial/{{pkg_name}}" $(ls -1t results/*.hart | head -1)
+done

oc-chef-pedant/lib/pedant.rb

@@ -16,6 +16,7 @@
 #Override the connect() method to set appropriate IPV6 host headers
 require 'pedant/core_ext/net_http'
 
+require 'uri'
 require 'pp' # Debugging
 
 require 'rspec'
@@ -48,6 +49,13 @@ def self.config
     Encoding.default_external = Encoding::UTF_8
     Encoding.default_internal = Encoding::UTF_8
 
+    # The URI gets normalized many places in the chain from pedant to erchef; in particular redundant port
+    # specifications (e.g 443 for https) are stripped out.  We normalize the URI here to make sure that the
+    # specs we check against conform to that requirement.
+    if Config.has_key?(:chef_server)
+      # chomp is to strip the trailing slash, which while technically correct, is improperly handled when we construct our specs
+      Config[:chef_server] = URI.parse(Config[:chef_server]).normalize.to_s.chomp('/')
+    end
     Config
   end
 

src/bookshelf/habitat/config/database-migrations.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+
+# TODO: not sure how to handle this. Sqitch bombs when it can't find the timezone
+export TZ="UTC"
+
+export PERL5LIB="$(cat "$(hab pkg path "chef-server/dbdpg")/LD_RUN_PATH")"
+export PATH=$PATH:$(hab pkg path "core/postgresql")/bin:$(hab pkg path "core/sqitch")/bin
+
+{{#if bind.database}}
+  {{#eachAlive bind.database.members as |member|}}
+    {{#if @last}}
+HOST="{{member.sys.ip}}"
+PORT="{{member.cfg.port}}"
+USER="{{member.cfg.superuser_name}}"
+PASS="{{member.cfg.superuser_password}}"
+DB="bookshelf"
+    {{/if}}
+  {{/eachAlive}}
+{{else}}
+HOST="{{cfg.postgresql.vip}}"
+PORT="{{cfg.postgresql.port}}"
+USER="{{cfg.sql_user}}"
+PASS="{{cfg.sql_password}}"
+DB="bookshelf"
+{{/if}}
+
+PG_ARGS="--host "$HOST" --port "$PORT" --username "$USER""
+export PGPASSWORD="$PASS"
+
+# Wait until postgres is ready
+until pg_isready $PG_ARGS --quiet; do :; done
+
+# Create delivery db for sqitch to deploy to
+createdb $PG_ARGS $DB "bookshelf"
+
+# Install uuid-ossp extension
+psql $PG_ARGS --command 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"' $DB
+
+cd "{{pkg.path}}/schema" || exit
+sqitch --quiet --engine pg deploy "db:pg://${USER}:${PASS}@${HOST}/$DB"

src/bookshelf/habitat/config/sys.config

@@ -0,0 +1,147 @@
+%% -*- mode: erlang -*-
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+
+[
+ {kernel, [{inet_dist_use_interface, {127,0,0,1}}]},
+ %% SASL config
+ {sasl, [
+         {sasl_error_logger, {file, "{{pkg.svc_var_path}}/logs/sasl-error.log"}},
+         {errlog_type, error},
+         {error_logger_mf_dir, "{{pkg.svc_var_path}}/logs"},      % Log directory
+         {error_logger_mf_maxbytes, 104857600},
+         {error_logger_mf_maxfiles, 10}
+        ]},
+ {lager, [
+          %% What handlers to install with what arguments
+          %% The defaults for the logfiles are to rotate the files when
+          %% they reach 10Mb or at midnight, whichever comes first, and keep
+          %% the last 5 rotations. See the lager README for a description of
+          %% the time rotation format:
+          %% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation
+          %%
+          %% If you wish to disable rotation, you can either set the size to 0
+          %% and the rotation time to "", or instead specify a 2-tuple that only
+          %% consists of {Logfile, Level}.
+          {handlers, [
+              {lager_console_backend, [{{cfg.lager.lager_console_backend}}, {lager_default_formatter, [ "[", severity, "] ", message, "\n"]}]},
+              {lager_file_backend, [
+                                    {file, "{{pkg.svc_var_path}}/logs/error.log"},
+                                    {level, error},
+                                    {size, 104857600},
+                                    {date, "$D0"},
+                                    {count, 10}
+                                   ]},
+              {lager_file_backend, [
+                                    {file, "{{pkg.svc_var_path}}/logs/console.log"},
+                                    {level, info},
+                                    {size, 104857600},
+                                    {date, "$D0"},
+                                    {count, 10},
+                                    %% Removes the logging source, since currently
+                                    %% there's only one place in all of Bifrost that
+                                    %% we log from.  Also removes the PID from
+                                    %% the default message format.
+                                    {formatter_config, [date, " ", time, " [", severity, "] ", message, "\n"]}
+                                   ]}
+              ]},
+
+          %% Whether to write a crash log, and where.
+          %% Commented/omitted/undefined means no crash logger.
+          {crash_log, "{{pkg.svc_var_path}}/logs/crash.log"},
+
+          %% Maximum size in bytes of events in the crash log - defaults to 65536
+          {crash_log_msg_size, 65536},
+
+          %% Maximum size of the crash log in bytes, before its rotated, set
+          %% to 0 to disable rotation - default is 0
+          {crash_log_size, 104857600},
+
+          %% What time to rotate the crash log - default is no time
+          %% rotation. See the lager README for a description of this format:
+          %% https://github.com/basho/lager/blob/master/README.org
+          {crash_log_date, "$D0"},
+
+          %% Number of rotated crash logs to keep, 0 means keep only the
+          %% current one - default is 0
+          {crash_log_count, 10},
+
+          %% Whether to redirect error_logger messages into lager - defaults to true
+          {error_logger_redirect, true},
+
+          %% Bump up the "high-water mark" (default 50), which is the
+          %% number of messages per second allowed to come from
+          %% error_logger.  This is the same as used by
+          %% opscode-chef-mover, FWIW.
+          {error_logger_hwm, 1000}
+        ]},
+  {chef_secrets, [
+                 {provider, chef_secrets_env}
+                ]},
+ {bookshelf, [{ip, "0.0.0.0"},
+              {port, {{cfg.port}} },
+              {storage_type, sql},
+              {disk_store, "{{pkg.svc_data_path}}"}, %% prob obsolete, revisit
+              {sql_retry_count, {{cfg.bookshelf.sql_retry_count}} },
+              {sql_retry_delay, {{cfg.bookshelf.sql_retry_delay}} },
+              {abandoned_upload_cleanup_interval, {{cfg.bookshelf.abandoned_upload_cleanup_interval}} },
+              {deleted_data_cleanup_interval, {{cfg.bookshelf.deleted_data_cleanup_interval}} },
+              {stream_download, {{cfg.bookshelf.stream_download}} },
+              {log_dir, "{{pkg.svc_var_path}}/log"}
+             ]},
+ {sqerl, [
+           {db_driver_mod, sqerl_pgsql_client},
+           {ip_mode, [ ipv4 ] },
+{{#if bind.database ~}}
+  {{#eachAlive bind.database.members as |member| ~}}
+    {{~#if @last}}
+           {db_host, "{{member.sys.ip}}"},
+           {db_port, {{member.cfg.port}} },
+           {db_user, "{{member.cfg.superuser_name}}"},
+           {db_pass, "{{member.cfg.superuser_password}}" },
+    {{/if ~}}
+  {{/eachAlive ~}}
+{{~else}}
+           {db_host, "{{cfg.sqerl.db_host}}"},
+           {db_port, {{cfg.sqerl.db_port}} },
+           {db_user, "{{cfg.sqerl.db_user}}"},
+           {db_pass, "{{cfg.sqerl.db_pass}}"},
+{{~/if}}
+           {db_name, "{{cfg.sqerl.db_name}}" },
+           {idle_check, 10000},
+           {pooler_timeout, {{cfg.sqerl.db_pooler_timeout}} },
+           {db_timeout, {{cfg.sqerl.sql_db_timeout}} },
+           {prepared_statements, {bksw_sql, statements, [pgsql]}},
+           {column_transforms,
+                              [{<<"created_at">>,
+                                {sqerl_transformers, convert_YMDHMS_tuple_to_datetime}},
+                               {<<"updated_at">>,
+                                {sqerl_transformers, convert_YMDHMS_tuple_to_datetime}}]
+           }
+          ]},
+ {pooler, [
+           {pools, [
+{{~#with cfg/pooler/sqerl }}
+                    [{name, sqerl},
+                     {max_count, {{max_count}} },
+                     {init_count, {{init_count}} },
+                     {queue_max, {{queue_max}} },
+                     {start_mfa, {sqerl_client, start_link, []}}]
+{{~/with ~}}
+                   ]}
+           %%,{metrics_module, folsom_metrics}
+          ]},
+ {webmachine, [
+          {log_handlers, [
+%%% Log handling is disabled, because it is redundant (nginx also logs requests)
+%%% If debug logging is needed, this can be uncommented to start logging somewhat verbose logs
+%%%              {oc_wm_request_logger, [
+%%%                       {file, "/var/logs/opscode/bookshelf/requests.log"},
+%%%                       {file_size, 100},  %% Size in MB
+%%%                       {files, 5},
+%%%                       {annotations, [user, req_id]}
+%%%                       ]
+%%%               }
+                         ]
+          }]}
+].

src/bookshelf/habitat/config/veil-secrets.json

@@ -0,0 +1,7 @@
+{{~ #if bind.chef-server-ctl}}
+  {{~ #eachAlive bind.chef-server-ctl.members as |member|}}
+    {{~ #if @last}}
+{{toJson member.cfg.secrets}}
+    {{~ /if}}
+  {{~ /eachAlive}}
+{{~ /if}}