Adding Habitization - supersedes PR#1416

This is a branch copy of chef#1416 rebased and with all commits squashed and authors
attributed below.

and are functional.

Co-authored-by: Elliott Davis <edavis@chef.io>
Co-authored-by: Irving Popovetsky <irving@chef.io>
Co-authored-by: Mark Anderson <mark@chef.io>
Co-authored-by: Nolan Davidson <ndavidson@chef.io>
Co-authored-by: Thomas Cate <thomascate@gmail.com>

Signed-off-by: Jeremy J. Miller <jm@chef.io>

.gitignore

@@ -29,3 +29,7 @@ compile_commands.json
 *.tfstate
 *.tfstate.backup
 *_override.tf
+
+# Habitat
+results
+

docker-compose.yml

@@ -0,0 +1,128 @@
+# Configurable environment variables:
+# HAB_ORIGIN - denotes the docker origin (dockerhub ID)
+# VERSION -  the version identifier tag on the packages
+# AUTOMATE_ENABLED - enable the Automate data collector (true or false)
+# AUTOMATE_SERVER - the IP address or hostname of the Automate server
+# AUTOMATE_TOKEN - the token for the Automate server data collector
+# USER_ID - the user ID to use
+# GROUP_ID - the group ID to use
+
+version: '2.1'
+services:
+  postgresql:
+    image: chefdemo/postgresql:stable
+    environment:
+      HAB_POSTGRESQL: |
+        [superuser]
+        name = 'hab'
+        password = 'chefrocks'
+    volumes:
+      - postgresql-data:/hab/svc/postgresql/data:Z
+
+  chef-server-ctl:
+    image: ${HAB_ORIGIN:-chefserverofficial}/chef-server-ctl:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+    environment:
+      HAB_CHEF_SERVER_CTL: |
+        [chef_server_api]
+        ip = '${HOST_IP:-172.17.0.1}'
+        [secrets.data_collector]
+        token = "${AUTOMATE_TOKEN:-93a49a4f2482c64126f7b6015e6b0f30284287ee4054ff8807fb63d9cbd1c506}"
+
+  elasticsearch:
+    image: chefdemo/elasticsearch:stable
+    command: --peer postgresql
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - elasticsearch-data:/hab/svc/elasticsearch/data:Z
+
+  oc_id:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_id:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  bookshelf:
+    image: ${HAB_ORIGIN:-chefserverofficial}/bookshelf:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  oc_bifrost:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_bifrost:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind database:postgresql.default
+      --bind chef-server-ctl:chef-server-ctl.default
+
+  oc_erchef:
+    image: ${HAB_ORIGIN:-chefserverofficial}/oc_erchef:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind bookshelf:bookshelf.default
+      --bind oc_bifrost:oc_bifrost.default
+      --bind database:postgresql.default
+      --bind elasticsearch:elasticsearch.default
+      --bind chef-server-ctl:chef-server-ctl.default
+    environment:
+      HAB_OC_ERCHEF: |
+        [chef_authn]
+        keygen_cache_workers=2
+        keygen_cache_size=10
+        keygen_start_size=0
+        keygen_timeout=20000
+        [data_collector]
+        enabled = ${AUTOMATE_ENABLED:-false}
+        server = "${AUTOMATE_SERVER:-localhost}"
+        port = 443
+
+  chef-server-nginx:
+    image: ${HAB_ORIGIN:-chefserverofficial}/chef-server-nginx:${VERSION:-latest}
+    user: ${USER_ID:-42}:${GROUP_ID:-42}
+    cap_drop:
+    - NET_BIND_SERVICE
+    - SETUID
+    - SETGID
+    command: --peer postgresql
+      --bind oc_erchef:oc_erchef.default
+      --bind oc_bifrost:oc_bifrost.default
+      --bind oc_id:oc_id.default
+      --bind bookshelf:bookshelf.default
+      --bind elasticsearch:elasticsearch.default
+      --bind chef-server-ctl:chef-server-ctl.default
+    volumes:
+      - nginx-data:/hab/svc/chef-server-nginx/data:Z
+    ports:
+      - 80:8080
+      - 443:8443
+
+volumes:
+  postgresql-data:
+  elasticsearch-data:
+  nginx-data:

habitat_pkgs_build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# this script builds all the essential habitat packages for running Chef Server
+# additionaly, it exports them as a local docker image
+
+CHEF_SERVER_SRC='/src/src'
+
+for dir in dbdpg oc-id openresty-noroot nginx bookshelf chef-server-ctl oc_bifrost oc_erchef; do
+  cd $CHEF_SERVER_SRC/$dir
+  echo "[STATUS] building $dir"
+  build > /var/log/build-${dir}-$(date +%s).log
+  if [[ $dir =~ dbdpg ]]; then continue; fi
+  echo "[STATUS] exporting $dir pkg to docker daemon"
+  hab pkg export docker --non-root -i "chefserverofficial/{{pkg_name}}" $(ls -1t results/*.hart | head -1)
+done

oc-chef-pedant/lib/pedant.rb

@@ -16,6 +16,7 @@
 #Override the connect() method to set appropriate IPV6 host headers
 require 'pedant/core_ext/net_http'
 
+require 'uri'
 require 'pp' # Debugging
 
 require 'rspec'
@@ -48,6 +49,13 @@ def self.config
     Encoding.default_external = Encoding::UTF_8
     Encoding.default_internal = Encoding::UTF_8
 
+    # The URI gets normalized many places in the chain from pedant to erchef; in particular redundant port
+    # specifications (e.g 443 for https) are stripped out.  We normalize the URI here to make sure that the
+    # specs we check against conform to that requirement.
+    if Config.has_key?(:chef_server)
+      # chomp is to strip the trailing slash, which while technically correct, is improperly handled when we construct our specs
+      Config[:chef_server] = URI.parse(Config[:chef_server]).normalize.to_s.chomp('/')
+    end
     Config
   end
 

omnibus/files/private-chef-ctl-commands/Gemfile

@@ -11,3 +11,7 @@ gem "veil", git: "https://github.com/chef/chef_secrets.git"
 
 # Dependencies of ctl commands
 gem "redis"
+gem "sequel"
+gem "rest-client"
+gem 'pg'
+gem 'knife-opc'

omnibus/files/private-chef-ctl-commands/chef-server-ctl

@@ -19,9 +19,15 @@ module Omnibus
       d
     end
 
+    CREDENTIAL_ENV = 'CHEF_SECRETS_DATA'.freeze
+
     def credentials
-      secrets_file = ENV['SECRETS_FILE'] || "/etc/opscode/private-chef-secrets.json"
-      @credentials ||= Veil::CredentialCollection::ChefSecretsFile.from_file(secrets_file)
+      if ENV.has_key?(CREDENTIAL_ENV)
+         @credentials = Veil::CredentialCollection::ChefSecretsEnv.new(var_name: CREDENTIAL_ENV)
+      else
+        secrets_file = ENV['SECRETS_FILE'] || "/etc/opscode/private-chef-secrets.json"
+        @credentials ||= Veil::CredentialCollection::ChefSecretsFile.from_file(secrets_file)
+      end
     end
 
     # Note that as we expand our external service support,

omnibus/files/private-chef-ctl-commands/chef12_upgrade_data_transform.rb

@@ -4,7 +4,7 @@
 # All Rights Reserved
 #
 
-require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+require_relative "open_source_chef12_upgrade"
 require 'optparse'
 require 'ostruct'
 

omnibus/files/private-chef-ctl-commands/chef12_upgrade_download.rb

@@ -4,7 +4,7 @@
 # All Rights Reserved
 #
 
-require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+require_relative "open_source_chef12_upgrade"
 require 'optparse'
 require 'ostruct'
 

omnibus/files/private-chef-ctl-commands/chef12_upgrade_upload.rb

@@ -4,7 +4,11 @@
 # All Rights Reserved
 #
 
-require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+begin
+  require_relative "open_source_chef12_upgrade"
+rescue LoadError
+  require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+end
 require 'optparse'
 require 'ostruct'
 

omnibus/files/private-chef-ctl-commands/key_control.rb

@@ -23,7 +23,7 @@
 # so we want to use the full path when loaded from omnibus-ctl,
 # but we need the local relative path for it to work with rspec
 begin
-  require "helpers/key_ctl_helper"
+  require_relative "helpers/key_ctl_helper"
 rescue LoadError
   require '/opt/opscode/embedded/service/omnibus-ctl/helpers/key_ctl_helper'
 end

omnibus/files/private-chef-ctl-commands/upgrade.rb

@@ -3,7 +3,11 @@
 # All Rights Reserved
 #
 
-require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+begin
+  require_relative "open_source_chef12_upgrade"
+rescue LoadError
+  require "/opt/opscode/embedded/service/omnibus-ctl/open_source_chef12_upgrade"
+end
 require 'optparse'
 require 'ostruct'
 

omnibus/files/private-chef-ctl-commands/wrap-knife-opc.rb

@@ -15,8 +15,16 @@
 
 require 'shellwords'
 
-knife_config = "/etc/opscode/pivotal.rb"
-knife_cmd    = "/opt/opscode/embedded/bin/knife"
+if File.exist?('/hab/svc/chef-server-ctl/PID')
+  knife_config = "/hab/svc/chef-server-ctl/config/pivotal.rb"
+  knife_path = `cat /hab/svc/chef-server-ctl/config/pkg_path`.chomp + "/chef/bin/knife"
+  bundler_path = `hab pkg path "core/bundler"`.chomp + "/bin/bundle"
+  knife_cmd =  "#{bundler_path} exec #{knife_path}"
+else
+  knife_config = "/etc/opscode/pivotal.rb"
+  knife_cmd    = "/opt/opscode/embedded/bin/knife"
+end
+
 cmd_args     = ARGV[3..-1]
 
 cmds = {

src/bookshelf/habitat/config/database-migrations.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+
+# TODO: not sure how to handle this. Sqitch bombs when it can't find the timezone
+export TZ="UTC"
+
+export PERL5LIB="$(cat "$(hab pkg path "chef-server/dbdpg")/LD_RUN_PATH")"
+export PATH=$PATH:$(hab pkg path "core/postgresql")/bin:$(hab pkg path "core/sqitch")/bin
+
+{{#if bind.database}}
+  {{#eachAlive bind.database.members as |member|}}
+    {{#if @last}}
+HOST="{{member.sys.ip}}"
+PORT="{{member.cfg.port}}"
+USER="{{member.cfg.superuser_name}}"
+PASS="{{member.cfg.superuser_password}}"
+DB="bookshelf"
+    {{/if}}
+  {{/eachAlive}}
+{{else}}
+HOST="{{cfg.postgresql.vip}}"
+PORT="{{cfg.postgresql.port}}"
+USER="{{cfg.sql_user}}"
+PASS="{{cfg.sql_password}}"
+DB="bookshelf"
+{{/if}}
+
+PG_ARGS="--host "$HOST" --port "$PORT" --username "$USER""
+export PGPASSWORD="$PASS"
+
+# Wait until postgres is ready
+until pg_isready $PG_ARGS --quiet; do :; done
+
+# Create delivery db for sqitch to deploy to
+createdb $PG_ARGS $DB "bookshelf"
+
+# Install uuid-ossp extension
+psql $PG_ARGS --command 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"' $DB
+
+cd "{{pkg.path}}/schema" || exit
+sqitch --quiet --engine pg deploy "db:pg://${USER}:${PASS}@${HOST}/$DB"