-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.go
156 lines (131 loc) · 2.94 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package main
import (
"log"
"sync"
)
type CrawlRequestPostData struct {
URL string `json:"url"`
UUID string `json:"uuid"`
}
type CrawlCompleteData struct {
UUID string
ScreenShot []byte
}
var chanCrawlRequestPostData chan *CrawlRequestPostData
var chanCrawlCompleteData chan *CrawlCompleteData
var chanWorkerDone chan int
var urlRequest mutexSlice
//totaal aantal chrome browsers die open mogen
var totalWorkers mutexCounter
var NumberPool numberPool
func main() {
log.Println("Starting PhishingEye")
//setup url request slice
urlRequest = mutexSlice{
mu: sync.Mutex{},
slice: make([]*CrawlRequestPostData, 0),
}
NumberPool = numberPool{
mu: sync.Mutex{},
pool: []int{},
}
for i := 0; i < 20; i++ {
NumberPool.ReleaseNumber(i + 9000)
}
//setup nieuw chan waar alle post data komt van de http server
chanCrawlRequestPostData = make(chan *CrawlRequestPostData, 100)
chanCrawlCompleteData = make(chan *CrawlCompleteData, 100)
chanWorkerDone = make(chan int, 20)
totalWorkers = mutexCounter{
mu: sync.Mutex{},
x: 1,
}
go func() {
for {
select {
case data := <-chanCrawlRequestPostData:
if totalWorkers.Value() > 0 && NumberPool.IsNumberAvailable() {
newCrawlFromHttp(data, NumberPool.GetNumber())
} else {
//voeg aan url request slice toe
urlRequest.AddData(data)
}
case data := <-chanCrawlCompleteData:
go SaveCrawlerData(data)
case number := <-chanWorkerDone:
totalWorkers.Add(1)
NumberPool.ReleaseNumber(number)
if urlRequest.Len() > 0 {
if NumberPool.IsNumberAvailable() {
newCrawlFromHttp(urlRequest.GetFirstElement(), NumberPool.GetNumber())
}
}
}
}
}()
NewHttpServer(3001)
}
func newCrawlFromHttp(data *CrawlRequestPostData, port int) {
totalWorkers.Subtract(1)
go CrawlURL(data.UUID, data.URL, port, chanWorkerDone)
}
type mutexCounter struct {
mu sync.Mutex
x int
}
func (c *mutexCounter) Subtract(x int) {
c.mu.Lock()
c.x -= x
c.mu.Unlock()
}
func (c *mutexCounter) Add(x int) {
c.mu.Lock()
c.x += x
c.mu.Unlock()
}
func (c *mutexCounter) Value() int {
return c.x
}
type mutexSlice struct {
mu sync.Mutex
slice []*CrawlRequestPostData
}
func (s *mutexSlice) AddData(data *CrawlRequestPostData) {
s.mu.Lock()
s.slice = append(s.slice, data)
s.mu.Unlock()
}
func (s *mutexSlice) GetFirstElement() *CrawlRequestPostData {
s.mu.Lock()
returnData := s.slice[0]
s.slice[0] = nil
s.slice = s.slice[1:]
s.mu.Unlock()
return returnData
}
func (s *mutexSlice) Len() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.slice)
}
type numberPool struct {
mu sync.Mutex
pool []int
}
func (n *numberPool) GetNumber() int {
n.mu.Lock()
returnData := n.pool[0]
n.pool = n.pool[1:]
n.mu.Unlock()
return returnData
}
func (n *numberPool) ReleaseNumber(i int) {
n.mu.Lock()
n.pool = append(n.pool, i)
n.mu.Unlock()
}
func (n *numberPool) IsNumberAvailable() bool {
n.mu.Lock()
defer n.mu.Unlock()
return len(n.pool) > 0
}