README_6SYNC_INSTALL

#
#  This is a log of the installation of a production server in early 2011.
#
#  It starts out fairly generic, and becomes increasingly specific to Mushroom
#  Observer.  Look for the following "variables:
#
#    IP_ADDR            ip address of new server
#    OTHER_IP_ADDR      ip address of old server
#    DOMAIN             mushroomobserver.org
#    SITE               mo
#    USER               jason
#    USER_PASSWORD      xxx
#    DATA_PASSWORD      xxx
#    MAIL_PASSWORD      xxx
#    $YOUR_EMAIL        pellaea@gmail.com
#    $FULL_NAME         Jason Hollinger
#
#  Tested from scratch, Mar 14, 2011. -Jason
#
#  (Many changes have been made since that test, though. I hope to test it
#  again in the next week or two, Mar 17, 2011. -Jason)
#
#  NOTE: I have added the yum equivalents to apt-get for Fedora / CentOS in
#  comments next to each apt-get line.  I can't guarantee these are all the ones
#  required.  Also note that a few configuration files will be in different
#  locations, and you will need to use chkconfig instead of update-rc.d.
#
#  ubuntu 10.10
#  ruby 1.9.2-p180
#  rubygems 1.6.2
#  passenger 3.0.5
#  apache2 2.2.16
#  postfix 2.7.1
#  mysql 14.14
#
################################################################################

# Initial set up.
local> ssh root@$IP_ADDR
root> passwd
root> apt-get upgrade
root> apt-get install tcsh mlocate man vim lynx telnet emacs wget \
                      build-essential iptables bison
root> # yum update
root> # yum install tcsh mlocate gcc make vim lynx telnet emacs bind-utils

# Create account for webserver.
root> useradd -m -s /bin/tcsh $SITE
root> passwd -l $SITE
root> visudo
  # Add these lines to end. Allows users in "mo" group to "sudo su mo".
  %$SITE ALL = NOPASSWD: /bin/su $SITE
  %$SITE ALL = NOPASSWD: /bin/su - $SITE

# Create account for yourself.
root> useradd -m -G $SITE -s /bin/tcsh $USER
root> passwd $USER_PASSWORD
root> logout

# Password-less ssh.
local> ssh-keygen -t rsa   # (if not already done)
local> ssh-copy-id -i ~/.ssh/id_rsa.pub $IP_ADDR

# Set up your user account however you like.
local> ssh $IP_ADDR
user> vi .tcshrc
  alias root sudo
  alias cls clear
  alias cd.. cd ..
  alias dirf ls -alF
  alias dirt ls -alFrt
  alias md mkdir
  alias rd rmdir
  alias del rm
  alias mv mv -i
  alias cp cp -i
  alias grep grep -s
  alias +x chmod a+x
  alias -x chmod a-x
  alias $SITE 'cd /var/web/$SITE; sudo su - $SITE'
user> vi .vimrc
  syntax off
  set nohlsearch
  set ts=2
  set sw=2
  set et
  set nowrap
user> source .tcshrc

# Set up root's and website's accounts.
user> su root
root> cp .tcshrc /root
root> cp .tcshrc /home/$SITE
root> cp .vimrc /root
root> cp .vimrc /home/$SITE
root> chmod 660 /home/*/.tcshrc
root> vi /root/.tcshrc
  # Add this line to improve performance of custom-built packages.
  # (Check lscpu and /proc/cpuinfo for cpu type.)
  setenv CFLAGS "-O3 -m64 -march=core2"
root> vi /home/$SITE/.tcshrc
  # Add these aliases to the end.
  alias $SITE 'cd /var/web/$SITE'
  alias ${SITE}sql  "mysql -u $SITE -p'$DATA_PASSWORD' ${SITE}_production"
  alias ${SITE}sqle "mysql -u $SITE -p'$DATA_PASSWORD' ${SITE}_production -e 'source \!^'"
root> vi /home/$SITE/.login
  setenv TZ 'Eastern Time (US & Canada)'
  setenv RAILS_ENV production
  cd /var/web/$SITE
root> chown -R $SITE:$SITE /home/$SITE
root> tcsh

# Disable direct root login for security purposes.
root> vi /etc/ssh/sshd_config
  PermitRootLogin no

# Configure firewall.
root> iptables -F
root> iptables -P INPUT ACCEPT
root> iptables -P OUTPUT ACCEPT
root> iptables -P FORWARD ACCEPT
root> iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
root> iptables -A INPUT -i lo -j ACCEPT
root> iptables -A INPUT -p icmp -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p tcp --dport 22   -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p tcp --dport 80   -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p tcp --dport 25   -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p tcp --dport 3306 -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p tcp --dport 53   -j ACCEPT
root> iptables -A INPUT -m state --state NEW -p udp --dport 53   -j ACCEPT
root> iptables -A INPUT   -j REJECT --reject-with icmp-host-prohibited
root> iptables -A FORWARD -j REJECT --reject-with icmp-host-prohibited
root> iptables-save > /etc/firewall.conf
root> chmod 600 /etc/firewall.conf
root> vi /etc/rc.local
  iptables-restore < /etc/firewall.conf

# Install most of the modules that we will need (except ruby).
root> apt-get install apache2 libapache2-mod-passenger \
        mysql-server mysql-client libmysqlclient-dev \
        subversion subversion-tools \
        libcurl4-openssl-dev libssl-dev libopenssl-ruby \
        apache2-prefork-dev libapr1-dev libaprutil1-dev
root> # yum install httpd-devel mysql-devel mysql-server svn curl-devel

# Build and install ruby from source to get good version.
root> cd
root> mkdir build
root> cd build
root> wget http://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.2-p180.tar.bz2
root> tar -xjf ruby*.bz2
root> cd ruby-1.9.2-p180
root> ./configure --with-ruby-version=1.9.2
root> make install
root> rehash
root> cd

# Ditto for rubygems.
root> cd ~/build
root> wget http://production.cf.rubygems.org/rubygems/rubygems-1.6.2.tgz
root> tar -xzf rubygems*.tgz
root> cd rubygems-1.6.2
root> ruby setup.rb --prefix=/usr/local --verbose
root> rehash
root> cd

# Now we can install ruby gems properly.
root> gem install rails --version=2.1.1 --verbose --no-rdoc --no-ri
root> gem install mysql2 --verbose --no-rdoc --no-ri
root> gem install test-unit --verbose --no-rdoc --no-ri
root> gem install RedCloth --verbose --no-rdoc --no-ri

# Install passenger.
root> gem install passenger --verbose --no-rdoc --no-ri
root> rehash
root> passenger-install-apache2-module
  # Pay attention to output at end and make sure values below are right.
root> vi /etc/apache2/mods-available/passenger.conf
  # Change root to /usr/local/lib/ruby/gems/1.9.2/gems/passenger-3.0.5
  # Change ruby to /usr/local/bin/ruby
root> vi /etc/apache2/mods-available/passenger.load
  # Change to /usr/local/lib/ruby/gems/1.9.2/gems/passenger-3.0.5/ext/apache2/mod_passenger.so
root> service apache2 stop
root> service apache2 start
# (For some reason reload and restart don't always work.)

# Install ImageMagick.
root> apt-get install imagemagick libmagickcore-dev libmagickwand-dev
root> # yum install ImageMagick-devel
root> gem install rmagick --verbose --no-rdoc --no-ri
root> gcc /var/web/mo/script/jpegresize.c -ljpeg -lm -O2 -o /usr/local/bin/jpegresize

# Create directory for rails app.
root> mkdir /var/web
root> mkdir /var/web/$SITE
root> mkdir /var/web/$SITE/{tmp,log}
root> touch /var/web/$SITE/tmp/restart.txt
root> chown -R $SITE:$SITE /var/web/$SITE
root> chmod 750 /var/web/$SITE

# Make sure apache user has access to rails directories.
root> usermod -a -G $SITE www-data

--------------------------------------------------------------------------------

# Now we're done with the generic installation.  Start installing Mushroom
# Observer in particular.

# Install rails apps.
root> mo
mo> svn co http://svn.collectivesource.com/mushroom_sightings/trunk/ .
mo> mkdir public/images/{thumb,320,640,960,1280,orig}
mo> vi config/consts-site.rb
  # Override these defaults.
  DOMAIN         = 'mushroomobserver.org'
  HTTP_DOMAIN    = 'http://mushroomobserver.org'
  SERVER_CODE    = 'us1'
  DEFAULT_LOCALE = 'en-US'
  BAD_DOMAINS    = ["www.#{DOMAIN}"]

  # Use queued email mechanism.
  QUEUE_EMAIL = true

  # Nathan wants to be BCC'ed on every single email.
  EXTRA_BCC_EMAIL_ADDRESSES = "mo@collectivesource.com"

  # Date after which votes become public.
  VOTE_CUTOFF = '20100405'

  # Set timezone?
  ENV['TZ'] = 'Eastern Time (US & Canada)'

  # Use CDMR mail server for ActionMailer.
  config.action_mailer.smtp_settings = {
    :address        => 'mail.digitalmycology.com',
    :port           => 587,
    :domain         => 'digitalmycology.com',
    :authentication => :login,
    :user_name      => 'mo@digitalmycology.com',
    :password       => '$MAIL_PASSWORD',
  }
mo> exit

# Configure apache.
root> vi /etc/apache2/sites-available/mo
  <VirtualHost *:80>
    ServerName mushroomobserver.org
    DocumentRoot /var/web/mo/public
    RailsEnv production
    SetEnv TZ 'Eastern Time (US & Canada)'

    # Chris Parrish might need this for his Chrome browser to work right?
    Header set Accept-Ranges none

    # This was necessary for my local installation on Fedora Core 13.
    # RailsBaseURI /

    <Directory /var/web/mo/public>
      Options FollowSymLinks
      AllowOverride limit
      Order allow,deny
      Allow from all
    </Directory>

    RewriteEngine on

    # Redirect Chris Parrish's lichens pages at CDMR.
    RewriteRule ^/lichens/(.*)$ http://lichens.digitalmycology.com/$1 [QSA]

    # Explicitly tell apache where to find static content in /public.
    # (Applies to several specific subdirs and any URL like /blah.ext.)
    # (This is apparently only required for the old fedora installation?)
    # RewriteRule ^/(stylesheets|javascripts|[\w\-\.]+\.\w+)(.*)$ /public/$1$2 [QSA]

    # Redirect all but thumbnails to image server.
    # RewriteRule ^/images/thumb/(.*)$ http://images.digitalmycology.com/thumb/$1 [QSA]
    # RewriteRule ^/images/320/(.*)$ http://images.digitalmycology.com/320/$1 [QSA]
    RewriteRule ^/images/640/(.*)$ http://images.digitalmycology.com/640/$1 [QSA]
    RewriteRule ^/images/960/(.*)$ http://images.digitalmycology.com/960/$1 [QSA]
    RewriteRule ^/images/1280/(.*)$ http://images.digitalmycology.com/1280/$1 [QSA]
    RewriteRule ^/images/orig/(.*)$ http://images.digitalmycology.com/orig/$1 [QSA]
  </VirtualHost>
root> a2dissite default
root> a2ensite mo
root> a2enmod rewrite
root> a2enmod headers

# Create database.
root> mo
mo> mysql -u root -p
  create database mo_production;
  create user 'mo'@'localhost' identified by '$DATA_PASSWORD';
  # set password for 'mo'@'localhost' = password('$DATA_PASSWORD');
  grant all privileges on mo_production.* to 'mo'@'localhost' with grant option;
mo> cp config/database.yml-template config/database.yml
mo> vi config/database.yml
  # Change these values in production section: (other sections aren't needed)
  adapter:  mysql2
  database: mo
  password: $DATA_PASSWORD
  socket:   /var/run/mysqld/mysqld.sock    # ubuntu/debian
  # socket:   /var/lib/mysql/mysql.sock    # fedora/centos?
  encoding: utf8
mo> cd db
mo> scp $USER@$OLD_SERVER:/var/web/mo/db/checkpoint.gz .
mo> gunzip checkpoint.gz
mo> mosqle checkpoint

# Set up password-less ssh for image transfers.
mo> ssh-keygen -t rsa
  # Just press return three times, tell it to overwrite id_rsa.pub if it asks.
mo> ssh-copy-id -i ~/.ssh/id_rsa.pub cdmr@images.digitalmycology.com
  # Enter password.
mo> ssh cdmr@images.digitalmycology.com
  # (just make sure it works)

--------------------------------------------------------------------------------

# This section deals with setting up email.  It's mostly generic.

# Set up simple mailserver using postfix.
root> apt-get install postfix mutt
  # Select "internet site" when it prompts you.
  # Enter the proper hostname as "system mail name" (should resolve back with reverse DNS).
root> # yum install postfix mutt
root> vi /etc/aliases
  # Change where root's mail goes.
  root: $USER,...
root> newaliases

# Test mail server.
root> su $USER
user> cd
user> echo $YOUR_EMAIL > .forward
user> vi .muttrc
  set realname="$FULL_NAME"
  set from="$USER@$DOMAIN"
  set use_from=yes
  set envelope_from=yes
user> echo "This is a test." | mutt -s test $YOUR_EMAIL
  # (Check /var/log/mail.log and /var/log/mail.err.)
user> exit

# Configure mail for MO.
root> mo
mo> cd
mo> vi .muttrc
  set realname="Mushroom Observer"
  set from="no-reply@mushroomobserver.org"
  set use_from=yes
  set envelope_from=yes
mo> vi .forward
  root
mo> echo "This is another test." | mutt -s test $YOUR_EMAIL
mo> exit

# Configure mail for rails.
# Should come configured "out of the box".
# Ask yourself a question via the website, e.g.:
#   http://$IP_ADDR/observer/ask_user_question/252
# Probably want to tail -f log/production.log if it's throwing errors.

# Configure no-reply to bounce correctly.
root> useradd -m no-reply
root> passwd -l no-reply
root> cp /var/web/mo/script/autoreply          /usr/local/bin/autoreply
root> cp /var/web/mo/config/no-reply.muttrc    /home/no-reply/.muttrc
root> cp /var/web/mo/config/no-reply.autoreply /home/no-reply/.autoreply
root> cp /var/web/mo/config/no-reply.forward   /home/no-reply/.forward
root> chmod 644 /home/no-reply/.[maf]*
root> chmod 755 /usr/local/bin/autoreply

--------------------------------------------------------------------------------

# Now the site should be mostly working.  We need to do some configuration to
# get image uploads to work, though.

# Make sure these have the correct image server(s).
mo> vim script/process_image
mo> vim script/update_images

# Test image upload on server:
#   http://$IP_ADDR/observer/create_observation
# Might need to run process_image once by hand before it will be fully automatic.
mo> ls public/images/orig
mo> script/process_image NNNNN jpg 0

# Grab thumbnails from image server.  (This takes hours!)
mo> script/update_images -v --sync dreamhost --download --thumb --320

--------------------------------------------------------------------------------

# Install cron jobs.

# They're all already present, just need to stick them in the crontab.
mo> crontab -e
  */5 * * * * /var/web/mo/script/run_rake email:send
  23  * * * * /var/web/mo/script/parse_log
  13  3 * * * /var/web/mo/script/run_script refresh_name_lister_cache
  43  3 * * * /var/web/mo/script/run_script update_lichen_list
  13  4 * * * /var/web/mo/script/run_script update_images --clean
  33 21 * * * /var/web/mo/script/makexml /var/web/mo/public/eol.xml

--------------------------------------------------------------------------------

# This section deals with creating all the additional user accounts for
# members of CDMR to give them access to the server in case of emergency.

# Create an email to send to CDMR members explaining what we're doing.
root> cd
root> cat > password_email <<EOB
      I've created an account on the new mushroomobserver.org server for you.
      Please log in using an ssh (Secure Shell) client and change your
      password at your earliest convenience: 

        ssh USER@$IP_ADDR
        (password = PASS)

      If you don't have an ssh client, or if you have any problems logging
      in, please let me know by replying to this email. 

      As a member of CDMR, you should have already received a copy of the
      administrative details for Mushroom Observer, including the root
      password for this new server.  However, for security reasons, we do not
      allow direct ssh log-in as root, so in the unlikely event that you need
      to access the server, you will need to login using your individual
      account, instead.  Once logged in you can switch to root via "su root". 

      If you prefer, you do not need to do any of this (let me know), but we
      would like at least a few members of CDMR to maintain active accounts
      on our servers, in case of emergency. 

      Thank you for your time,
      Jason Hollinger
      Mushroom Observer System Administrator
      EOB

# Create user accounts, initialize with random password, mail password to root.
root> foreach user (anne darv mgwood mike velosa)
        set pass=`cat /dev/urandom | tr -dc A-Za-z0-9_ | head -c8`
        useradd -m -s /bin/tcsh $user
        echo "$user":"$pass" | chpasswd
        passwd -e $user
        cat password_email | sed "s/USER/$user/g" | sed "s/PASS/$pass/" | \
          mail -s "account on mushroomobserver.org" root
      end

# If you need to remove a user completely (e.g., a test user).
# root> userdel -r $user

# To copy over login information from another server instance.
# (NOTE: this doesn't seem to work going from Fedora to Ubuntu,
# it may only work if both are running the same distribution.)
# root> vi /etc/shadow
#   # (copy lines from the same file on the other server)

--------------------------------------------------------------------------------

# This section deals with sharing databases between multiple servers.
# NOTE: This is also useful for running two servers in parallel while testing
# a new installation

# Configure database to accept remote connections.
root> vi /etc/mysql/my/cnf
  # Comment out this line to allow remote connections.
  # bind-address = 127.0.0.1
root> mysql -u root -p
  create user 'mo'@'$OTHER_IP_ADDR' identified by '$DATA_PASSWORD';
  grant all privileges on mo_production.* to 'mo'@'$OTHER_IP_ADDR' with grant option;
root> ssh $USER@$OTHER_IP_ADDR
user> mysqladmin --protocol=tcp -u mo -p'$DATA_PASSWORD' --host=$IP_ADDR ping
user> exit

# To switch databases from old server to new server.
# First set up new server as above.
# Make sure everyone can connect to the new server's database.

# Shut down apache2 on all servers.
xxx_root> service apache2 stop

# Make snapshot of old database.
old_root> mo
old_mo> cd db
old_mo> ./dump

# Copy snapshot to new server.
new_root> mo
new_mo> cd db
new_mo> scp $USER@$OLD_IP_ADDR:/var/web/mo/db/checkpoint.gz .
new_mo> gunzip checkpoint.gz
new_mo> mosqle checkpoint

# Reconfigure and restart all servers.
xxx_root> mo
xxx_mo> vi config/database.yml
  production:
    database: mo_production
    host:     $IP_ADDR
    username: mo
    password: $DATA_PASSWORD
xxx_mo> rake db:version   # (just a simple test)
xxx_mo> exit
xxx_root> service apache2 start

--------------------------------------------------------------------------------

# A few post-installation sanity-checks.  Reboot after everything is done,
# then check the following.

# Make sure firewall is working.  If there are more than a few lines it is
# probably correct, but you can also compare it with another server.
root> iptables-save

# Make sure all the critical processes started automatically.
# Look for sshd, apache2, Passenger, mysqld, postfix.
root> ps -ef

# If any are missing, do this on ubuntu/debian:
root> update-rc.d apache2 defaults
root> update-rc.d mysql   defaults
root> update-rc.d postfix defaults

# Or this on fedora/centos:
# root> chkconfig --level 345 httpd   on
# root> chkconfig --level 345 mysqld  on
# root> chkconfig --level 345 postfix on