-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathnode_config_scripts.sh
145 lines (108 loc) · 4.29 KB
/
node_config_scripts.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# This file contains all the commands and other instructions used for creating the cluster
#////////////In your home machine///////////////#
# Change the permissions of the private key to 0600
sudo chmod 600 ~/.ssh/group10keypair.pem
#Create a ~/.ssh/config file in order to not to need to type -i [private_key_name] every time
touch ~/.ssh/config
nano ~/.ssh/config
# Add the following to the file
Host 130.238.28.124 #The floating IP address or the host name of the machine.
IdentityFile ~/.ssh/group10keypair.pem
# SSH:ing to group101 node
ssh -L 8888:localhost:8888 -L 8080:localhost:8080 -L 4040:localhost:4040 ubuntu@130.238.28.124 -X
#////////////Inside each node///////////////#
# Check what is the hostname and set it
hostname
sudo nano /etc/hosts
# It becomes in this case:
# 127.0.0.1 group101
sudo apt-get update
sudo apt-get upgrade
# Install jdk
sudo apt-get install default-jdk
# Checking that Java is installed correctly
java -version
which java
readlink -f /usr/bin/java
#install pip
sudo apt-get install python3-pip
#Install Eye of Gnome, so we can view image files (plots)
#Have to log in using $ssh -i [KEY] user@ip -X
#The key being the "-X". This uses your own display as output-display
sudo apt-get install eog
# Fixing locale not set error
# LC_ALL = (unset)
# perl: warning: Setting locale failed.
echo "export LANGUAGE=en_US.UTF-8
export LANG=en_US.UTF-8
export LC_ALL=en_US.UTF-8">>~/.profile
source ~/.profile
# Upgrade pip to the latest version
python3 -m pip install --upgrade pip
# Double check that the version is 10.0.1
python3 -m pip --version
# Download Spark into home directory
wget http://apache.mirrors.spacedump.net/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz
# Uncompress the Spark file
tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz
# Remove the compressed file
rm spark-2.3.0-bin-hadoop2.7.tgz
# Install jupyter
sudo python3 -m pip install jupyter
# Install matplotlib
sudo python3 -m pip install matplotlib
# Add these lines to the end of ~/.bashrc
#Spark
export SPARK_HOME="/home/<your_username>/spark-2.3.0-bin-hadoop2.7/"
# Adding environment variables
echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/
export SPARK_HOME=/home/ubuntu/spark-2.3.0-bin-hadoop2.7
export PATH=$PATH:/home/ubuntu/bin:/home/ubuntu/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/bin:/home/ubuntu/spark-2.3.0-bin-hadoop2.7/bin
export PYTHONPATH=/python:/python/lib/py4j-0.10.6-src.zip
export PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.6-src.zip:$PYTHONPATH
export PYSPARK_PYTHON=python3">>~/.profile
#update the .profile
source ~/.profile
# Making each node aware of each other
sudo nano /etc/hosts
#Change the hostname to the correct one
# Add these lines to the end
192.168.1.191 group101
192.168.1.166 group102
192.168.1.162 group103
192.168.1.163 group104
192.168.1.167 group105
192.168.1.174 group106
# Disable firewall
sudo ufw disable # Not implemented in node: "group101" though
# In each node
cd ~/.ssh/
# Download the private key to each node
# Note the link might not work anymore
wget https://filedn.com/lMeFQXHqV237y22exG4jHzQ/group10keypair.pem
# Change the permissions
sudo chmod 600 ~/.ssh/group10keypair.pem
# Test to connect to master node (group101)
ssh -i ~/.ssh/group10keypair.pem ubuntu@130.238.28.124
# Make a source directory where the data is placed
mkdir data
# Download the data into the created directory
wget https://filedn.com/lMeFQXHqV237y22exG4jHzQ/entries.idx
wget https://filedn.com/lMeFQXHqV237y22exG4jHzQ/pdb_seqres.txt
#////////////Setting up the cluster///////////////#
Created a snapshot of instance configured as above
Created 5 more instances from the snapshot
#////////////Starting the cluster///////////////#
#////////////Inside the master///////////////#
# Start the master
./spark-2.3.0-bin-hadoop2.7/sbin/start-master.sh -h 192.168.1.191
# Stop the master
./spark-2.3.0-bin-hadoop2.7/sbin/stop-master.sh
#////////////////////////////////////////////#
# SSH:ing into each worker
ssh -i ~/.ssh/group10keypair.pem ubuntu@192.168.1.166
#////////////Inside the slaves///////////////#
# Run this command to start the slave
./spark-2.3.0-bin-hadoop2.7/sbin/start-slave.sh spark://192.168.1.191:7077
# In order to stop the slave run this command
./spark-2.3.0-bin-hadoop2.7/sbin/stop-slave.sh