-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: 新增流控特性和简单监控特性，分割特性最大值最小值预置值 (#13)
* feat: 新增流控特性 * feat & docs: 新增channel统计信息,变更reader，writer的rdbm到dbms, 新增切分主键预置最大值和最小值 * feat& fix: 新增流量监控和修复限流特性中的问题 * feat & test: 修复打印结果和测试1.19,1.20 * test & fix: 修复通道数无法生效的问题，测试split的默认值。 * fix: 调整通道数为4
- Loading branch information
1 parent
14665f4
commit c26eb4e
Showing
104 changed files
with
1,386 additions
and
473 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| { | ||
| "core" : { | ||
| "container": { | ||
| "job":{ | ||
| "id": 1, | ||
| "sleepInterval":100 | ||
| } | ||
| } | ||
| }, | ||
| "job":{ | ||
| "content":[ | ||
| { | ||
| "reader":{ | ||
| "name": "csvreader", | ||
| "parameter": { | ||
| "path":["examples/limit/src.csv"], | ||
| "encoding":"utf-8", | ||
| "delimiter":"," | ||
| } | ||
| }, | ||
| "writer":{ | ||
| "name": "csvwriter", | ||
| "parameter": { | ||
| "path":["examples/limit/dest.csv"], | ||
| "encoding":"utf-8", | ||
| "delimiter":"," | ||
| } | ||
| }, | ||
| "transformer":[] | ||
| } | ||
| ], | ||
| "setting":{ | ||
| "speed":{ | ||
| "byte":1024, | ||
| "record":100, | ||
| "channel":4 | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // Copyright 2020 the go-etl Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "encoding/base64" | ||
| "encoding/csv" | ||
| "fmt" | ||
| "os" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| "github.com/Breeze0806/go-etl/element" | ||
| ) | ||
|
|
||
| func main() { | ||
| f, err := os.Create("src.csv") | ||
| if err != nil { | ||
| fmt.Println("crete file fail. err:", err) | ||
| return | ||
| } | ||
| defer f.Close() | ||
|
|
||
| w := csv.NewWriter(f) | ||
| for i := 0; i < 1000000; i++ { | ||
| record := []string{strconv.Itoa(i), | ||
| time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, i/1000).Format(element.DefaultTimeFormat[:10]), | ||
| base64.StdEncoding.EncodeToString([]byte{byte(i / 100 / 100), byte((i / 100) % 100), byte(i % 100)}), | ||
| } | ||
| w.Write(record) | ||
| if (i+1)%1000 == 0 { | ||
| w.Flush() | ||
| } | ||
| } | ||
| w.Flush() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.