Use pam_namespaces and pam_exec to clean-up home dirs and temp space

Also DRY-up sysctl/pam code repeated in Kafka and Hadoop; should be in BCPC

cookbooks/bcpc-hadoop/metadata.rb

@@ -12,7 +12,6 @@
 depends 'database'
 depends 'java'
 depends 'poise'
-depends 'pam'
 depends 'sysctl'
 depends 'ulimit'
 depends 'locking_resource'

cookbooks/bcpc-hadoop/recipes/configs.rb

@@ -2,8 +2,6 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 
 include_recipe 'bcpc-hadoop::default'
-# NOTE: This include_recipe is necessary for resource collection
-include_recipe 'sysctl::default'
 
 # disable IPv6 (e.g. for HADOOP-8568)
 case node['platform_family']
@@ -19,29 +17,6 @@
   Chef::Log.warn '============ Unable to disable IPv6 for non-Debian systems'
 end
 
-# ensure we use /etc/security/limits.d to allow ulimit overriding
-if !node.key?('pam_d') || !node['pam_d'].key?('services') || !node['pam_d']['services'].key?('common-session')
-  node.default['pam_d']['services'] = {
-    'common-session' => {
-      'main' => {
-        'pam_permit_default' => { 'interface' => 'session', 'control_flag' => '[default=1]', 'name' => 'pam_permit.so' },
-        'pam_deny' => { 'interface' => 'session', 'control_flag' => 'requisite', 'name' => 'pam_deny.so' },
-        'pam_permit_required' => { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_permit.so' },
-        'pam_limits' => { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_limits.so' },
-        'pam_umask' => { 'interface' => 'session', 'control_flag' => 'optional', 'name' => 'pam_umask.so' },
-        'pam_unix' => { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_unix.so' }
-      },
-      'includes' => []
-    }
-  }
-end
-
-# set vm.swapiness to 0 (to lessen swapping)
-# NOTE: See above for note about resource collection
-sysctl_param 'vm.swappiness' do
-  value 0
-end
-
 # Populate node attributes for all kind of hosts
 set_hosts
 node.override['locking_resource']['zookeeper_servers'] = \

cookbooks/bcpc-hadoop/recipes/datanode.rb

@@ -10,6 +10,9 @@
   'docopy' => true
 }
 
+# ensure we do not make polyinstantation directories for hdfs and yarn
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hdfs', 'yarn']
+
 hdp_select_pkgs = %w(hadoop-yarn-nodemanager hadoop-hdfs-datanode hadoop-client)
 
 hdp_pkg_strs = (hdp_select_pkgs + %w(

cookbooks/bcpc-hadoop/recipes/hannibal.rb

@@ -1,5 +1,7 @@
 # Setup hannibal config
 
+# ensure we do not make polyinstantation directories for hannibal
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hannibal']
 # Populate node attributes for all kind of hosts
 set_hosts
 

cookbooks/bcpc-hadoop/recipes/hbase_master.rb

@@ -3,6 +3,8 @@
 include_recipe 'bcpc-hadoop::hbase_config'
 include_recipe 'bcpc-hadoop::hbase_queries'
 
+# ensure we do not make polyinstantation directories for hbase
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hbase']
 #
 # Updating node attributes to copy HBase master log file to centralized location (HDFS)
 #

cookbooks/bcpc-hadoop/recipes/historyserver.rb

@@ -2,6 +2,9 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 ::Chef::Resource::Bash.send(:include, Bcpc_Hadoop::Helper)
 
+# ensure we do not make polyinstantation directories for mapred
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['mapred']
+
 %w{hadoop-mapreduce-historyserver}.each do |pkg|
   package hwx_pkg_str(pkg, node[:bcpc][:hadoop][:distribution][:release]) do
     action :install

cookbooks/bcpc-hadoop/recipes/hive_hcatalog.rb

@@ -6,6 +6,8 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 ::Chef::Resource::Bash.send(:include, Bcpc_Hadoop::Helper)
 
+# ensure we do not make polyinstantation directories for hive
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hive']
 %w{hadooplzo hadooplzo-native hive-hcatalog}.map do |pp|
   hwx_pkg_str(pp, node[:bcpc][:hadoop][:distribution][:release])
 end.each do |pkg|

cookbooks/bcpc-hadoop/recipes/httpfs.rb

@@ -3,6 +3,9 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 ::Chef::Resource::Bash.send(:include, Bcpc_Hadoop::Helper)
 
+# ensure we do not make polyinstantation directories for httpfs
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['httpfs']
+
 package hwx_pkg_str("hadoop-httpfs", node[:bcpc][:hadoop][:distribution][:release]) do
   action :install
 end

cookbooks/bcpc-hadoop/recipes/httpfs_config.rb

@@ -27,4 +27,4 @@
      source "#{t}.erb"
      mode 0644
   end
-end
+end

cookbooks/bcpc-hadoop/recipes/journalnode.rb

@@ -42,6 +42,9 @@
 
 end
 
+# ensure we do not make polyinstantation directories for hdfs
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hdfs']
+
 %w{hadoop-hdfs-namenode hadoop-hdfs-journalnode}.each do |pkg|
   package hwx_pkg_str(pkg, hdprel) do
     action :install

cookbooks/bcpc-hadoop/recipes/namenode_master.rb

@@ -24,6 +24,9 @@
   }
 end
 
+# ensure we do not make polyinstantation directories for hdfs
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hdfs']
+
 # shortcut to the desired HDFS command version
 hdfs_cmd = "/usr/hdp/#{node[:bcpc][:hadoop][:distribution][:active_release]}/"\
            'hadoop-hdfs/bin/hdfs'

cookbooks/bcpc-hadoop/recipes/namenode_no_HA.rb

@@ -18,6 +18,9 @@
     'docopy' => true
 }
 
+# ensure we do not make polyinstantation directories for hdfs
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hdfs']
+
 # shortcut to the desired HDFS command version
 hdfs_cmd = "/usr/hdp/#{node[:bcpc][:hadoop][:distribution][:active_release]}/hadoop-hdfs/bin/hdfs"
 

cookbooks/bcpc-hadoop/recipes/namenode_standby.rb

@@ -16,6 +16,9 @@
     'docopy' => true
 }
 
+# ensure we do not make polyinstantation directories for hdfs
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hdfs']
+
 # shortcut to the desired HDFS command version
 hdfs_cmd = "/usr/hdp/#{node[:bcpc][:hadoop][:distribution][:active_release]}/hadoop-hdfs/bin/hdfs"
 

cookbooks/bcpc-hadoop/recipes/oozie.rb

@@ -27,6 +27,9 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 ::Chef::Resource::Bash.send(:include, Bcpc_Hadoop::Helper)
 
+# ensure we do not make polyinstantation directories for oozie
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['oozie']
+
 #
 # These data bags and vault items are pre-populated at compile time by
 # the bcpc::mysql_data_bags recipe.

cookbooks/bcpc-hadoop/recipes/opentsdb.rb

@@ -35,4 +35,7 @@
 node.force_default['bach_opentsdb']['hbase_region_princ'] =
   "hbase/_HOST@#{krb_realm}"
 
+# ensure we do not make polyinstantation directories for opentsdb
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += [node['bach_opentsdb']['tsd_user']]
+
 include_recipe 'bach_opentsdb'

cookbooks/bcpc-hadoop/recipes/phoenix_query_server.rb

@@ -4,6 +4,9 @@
 
 qs_runas = node['bcpc']['hadoop']['phoenix']['phoenixqs']['username']
 
+# ensure we do not make polyinstantation directories for PQS
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += [qs_runas]
+
 user qs_runas do
   comment 'Runs phoenix queryserver'
   only_if { node['bcpc']['hadoop']['phoenix']['phoenixqs']['localuser'] }

cookbooks/bcpc-hadoop/recipes/region_server.rb

@@ -11,6 +11,9 @@
   copylog['region_server_out']['docopy'] = true
 end
 
+# ensure we do not make polyinstantation directories for hbase
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['hbase']
+
 (%w(libsnappy1) + %w(hbase hbase-regionserver phoenix).map do |p|
   hwx_pkg_str(p, node[:bcpc][:hadoop][:distribution][:release])
 end).each do |pkg|

cookbooks/bcpc-hadoop/recipes/resource_manager.rb

@@ -2,6 +2,9 @@
 ::Chef::Recipe.send(:include, Bcpc_Hadoop::Helper)
 ::Chef::Resource::Bash.send(:include, Bcpc_Hadoop::Helper)
 
+# ensure we do not make polyinstantation directories for YARN
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['yarn']
+
 ruby_block 'create-yarn-directories' do
   block do
     node.run_state['bcpc_hadoop_disks']['mounts'].each do |disk_number|

cookbooks/bcpc-hadoop/recipes/smoke_test_user.rb

@@ -20,6 +20,9 @@
 
 test_user = node['hadoop_smoke_tests']['oozie_user']
 
+# ensure we do not make polyinstantation directories for the Smoke Tests
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += [test_user]
+
 # create a local user and group if needed
 user test_user do
   manage_home true

cookbooks/bcpc-hadoop/recipes/yarnproxy.rb

@@ -8,6 +8,9 @@
   end
 end
 
+# ensure we do not make polyinstantation directories for the yarn
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['yarn']
+
 hdp_select('hadoop-mapreduce-historyserver', node[:bcpc][:hadoop][:distribution][:active_release])
 
 service "hadoop-yarn-proxyserver" do 

cookbooks/bcpc-hadoop/recipes/zookeeper_impl.rb

@@ -5,6 +5,9 @@
 
 include_recipe 'bcpc-hadoop::zookeeper_packages'
 
+# ensure we do not make polyinstantation directories for the zookeepers
+node.default['bcpc']['pam_namespace']['real_home_dir_users'] += ['zookeepers']
+
 user_ulimit 'zookeeper' do
   filehandle_limit 65_536
 end

cookbooks/bcpc/attributes/default.rb

@@ -250,3 +250,8 @@
 # sshd_config
 default['bcpc']['ssh']['address_family'] = 'inet'
 default['bcpc']['ssh']['x11_forwarding'] = false
+
+# pam config
+default['bcpc']['pam_namespace']['real_home_dir_users'] = ['root', 'ubuntu']
+default['bcpc']['pam_namespace']['shm_polyinstantion_dir'] = 'inst-dir'
+default['bcpc']['pam_namespace']['polyinstantion_dir'] = File.join('/', node['bcpc']['pam_namespace']['shm_polyinstantion_dir'])

cookbooks/bcpc/metadata.rb

@@ -22,4 +22,6 @@
 depends 'ntp'
 depends 'ubuntu'
 depends 'sudo'
+depends 'sysctl'
+depends 'pam'
 depends 'pdns'

cookbooks/bcpc/recipes/cronjobs.rb

@@ -18,9 +18,12 @@
 
 # Base cronjobs/ pseudo-cronjobs that should be on all machines in the cluster.
 
+polyinstantion_dir = node['bcpc']['pam_namespace']['polyinstantion_dir']
+shm_polyinstantion_dir = node['bcpc']['pam_namespace']['shm_polyinstantion_dir']
+
 clear_tmp = node['bcpc']['cronjobs']['clear_tmp']
-execute 'clear /tmp' do
-  command '/usr/bin/find /tmp -type f '\
+execute 'clear tmp dirs' do
+  command '/usr/bin/find /tmp #{polyinstantion_dir} /dev/shm/#{shm_polyinstantion_dir} -type f '\
           "-atime +#{clear_tmp['atime_age']} -delete && "\
           '/usr/bin/touch /var/lib/clear-temp.run'
   not_if do

cookbooks/bcpc/recipes/pam.rb

@@ -0,0 +1,86 @@
+#
+# Cookbook Name:: bcpc
+# Recipe:: pam
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Chef recipe to implement pam_namespace polyinstantiated directories
+# This will provide users the appearance they are the only user with data
+# in the affected directories -- also this will clean-up their data when
+# leaving the machine
+
+directory '/inst-dirs' do
+  user 'root'
+  group 'root'
+  mode 0o000
+end
+
+directory '/usr/local/sbin' do
+  action :create
+end
+
+polyinstantion_dir = node['bcpc']['pam_namespace']['polyinstantion_dir']
+shm_polyinstantion_dir = node['bcpc']['pam_namespace']['shm_polyinstantion_dir']
+
+template '/usr/local/sbin/inst_dir.sh' do
+  source 'inst_dir.sh.erb'
+  mode 500
+  variables(shm_polyinstantion_dir: shm_polyinstantion_dir,
+            polyinstantion_dir: polyinstantion_dir)
+end
+
+template '/etc/security/namespace.conf' do
+  source 'pam_namespace.conf.erb'
+  mode 500
+  variables(lazy {{ real_home_dir_users:
+                     node['bcpc']['pam_namespace']['real_home_dir_users'].join(','),
+                   shm_polyinstantion_dir: shm_polyinstantion_dir,
+                   polyinstantion_dir: polyinstantion_dir
+                 }})
+end
+
+template '/etc/security/namespace.init' do
+  source 'pam_namespace.init.erb'
+  mode 755
+end
+
+# NOTE: This include_recipe is necessary for resource collection
+include_recipe 'sysctl::default'
+
+# ensure we use /etc/security/limits.d to allow ulimit overriding
+if !node.key?('pam_d') || !node['pam_d'].key?('services') || !node['pam_d']['services'].key?('common-session')
+  node.default['pam_d']['services'] = {
+    'common-session' => {
+      'main' => {
+        'pam_permit_default' =>  { 'interface' => 'session', 'control_flag' => '[default=1]', 'name' => 'pam_permit.so' },
+        'pam_deny' =>            { 'interface' => 'session', 'control_flag' => 'requisite', 'name' => 'pam_deny.so' },
+        'pam_permit_required' => { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_permit.so' },
+        'pam_limits' =>          { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_limits.so' },
+        'pam_umask' =>           { 'interface' => 'session', 'control_flag' => 'optional', 'name' => 'pam_umask.so' },
+        'pam_unix' =>            { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_unix.so' },
+        'pam_exec' =>            { 'interface' => 'session', 'control_flag' => 'optional', 'name' => 'pam_exec.so', 'args' => '/usr/local/sbin/inst_dir.sh' },
+        'pam_namespace' =>       { 'interface' => 'session', 'control_flag' => 'required', 'name' => 'pam_namespace.so', 'args' => 'unmnt_remnt' },
+      },
+      'includes' => []
+    }
+  }
+end
+
+# set vm.swapiness to 0 (to lessen swapping)
+sysctl_param 'vm.swappiness' do
+  value 0
+end
+

cookbooks/bcpc/templates/default/inst_dir.sh.erb

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script is managed by Chef
+# It is designed to setup directories using pam_exec
+# for the pam_namespaces module to isolate user-direcotires
+
+if [ -z "$PAM_TYPE" -o "$PAM_TYPE" == "open_session" ]; then
+  # need to setup the polyinstantation directory for login
+  mkdir -pm 000 "/dev/shm/<%= @shm_polyinstantion_dir %>" \
+                "<%= @polyinstantion_dir %>"
+elif [ -z "$PAM_TYPE" -o "$PAM_TYPE" == "close_session" ]; then
+  # see if user still has any prescence on this machine
+  pgrep -u "$PAM_USER" && exit 0
+  # if no prescense of user, remove all trace of their activity
+  rm -rf "<%= @polyinstantion_dir %>/home_$PAM_USER" \
+         "<%= @polyinstantion_dir %>/tmp_$PAM_USER" \
+         "<%= @polyinstantion_dir %>/var_tmp_$PAM_USER" \
+         "/dev/shm/<%= @shm_polyinstantion_dir %>/inst_$PAM_USER"
+fi
+exit 0

cookbooks/bcpc/templates/default/pam_namespace.conf.erb

@@ -0,0 +1,11 @@
+# /etc/security/namespace.conf
+#
+#############################
+# This file managed by Chef #
+#############################
+#
+#masked mnt point   real mnt point               isolation-level    excluded users
+/var/tmp            <%= @polyinstantion_dir %>/var_tmp_           user              root
+$HOME               <%= @polyinstantion_dir %>/home_              user              <%= @real_home_dir_users %>
+/tmp                <%= @polyinstantion_dir %>/tmp_               user              root
+/dev/shm            /dev/shm/<%= @shm_polyinstantion_dir %>/inst_      user              root