-
Notifications
You must be signed in to change notification settings - Fork 0
/
pipeline.go
46 lines (37 loc) · 1.04 KB
/
pipeline.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
package crawler
// import "dcms"
// ItemPipelineFunc 处理item的函数
type ItemPipelineFunc func(item interface{}, ctx *Context) interface{}
// ItemPipeline pipeline接口
type ItemPipeline interface {
ProcessItem(item interface{}, ctx *Context) interface{}
}
type funcPipeline struct {
callback ItemPipelineFunc
}
func (cp *funcPipeline) ProcessItem(item interface{}, ctx *Context) interface{} {
if cp.callback != nil {
return cp.callback(item, ctx)
}
return item
}
// MongoPipeline 默认mongodb pipeline
type MongoPipeline struct {
MongoDBURI string
Database string
Collection string
}
// ProcessItem 实现ItemPipeline接口
func (dmp *MongoPipeline) ProcessItem(item interface{}, ctx *Context) interface{} {
// dcms.SaveItem(item)
return item
}
// DefaultPipeLines 默认的pipelines
func DefaultPipeLines() []ItemPipeline {
pipeline := MongoPipeline{}
return []ItemPipeline{&pipeline}
}
// FuncPipeline 仅提供一个函数的pipeline
func FuncPipeline(callback ItemPipelineFunc) ItemPipeline {
return &funcPipeline{callback}
}