Skip to content

Commit

Permalink
跳过表达式策略修改
Browse files Browse the repository at this point in the history 
1. 使用!time保留需要下载的分片时, 保留边界分片
2. 使用time剔除需要下载的分片时, 保留边界分片
  • Loading branch information
@orestonce
orestonce committed Oct 15, 2024
1 parent f9940ad commit 4ba0f3f
Show file tree
Hide file tree
Showing 6 changed files with 218 additions and 152 deletions.
8 changes: 4 additions & 4 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,20 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo)

this.status.SetProgressBarTitle("[2/5]获取ts列表")
tsList := info.GetTsList()
tsList, skipTsList := skipApplyFilter(tsList, skipInfo)
tsList, skipTsRecordList := skipApplyFilter(tsList, skipInfo)
if len(tsList) <= 0 {
this.setErrMsg("需要下载的文件为空")
return
}
// 获取m3u8地址的内容体
err = this.updateMedia(req.M3u8Url, tsList)
if err != nil {
this.setErrMsg("getEncryptInfo: " + err.Error())
this.setErrMsg("updateMedia: " + err.Error())
return
}

for _, ts := range skipTsList {
this.status.setTsNotWriteReason(&ts, "触发跳过表达式")
for _, record := range skipTsRecordList {
this.status.setTsNotWriteReason(&record.ts, "触发跳过表达式,"+record.reason)
}

// 下载ts
Expand Down
17 changes: 2 additions & 15 deletions download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,6 @@ import (
"github.com/orestonce/gopool"
)

// TsInfo 用于保存 ts 文件的下载地址和文件名
type TsInfo struct {
Name string
Url string
Seq uint64 // 如果是aes加密并且没有iv, 这个seq需要充当iv
TimeSec float64 // 此ts片段占用多少秒
Between_EXT_X_DISCONTINUITY bool
SkipByHttpCode bool
HttpCode int
}

type GetStatus_Resp struct {
Percent int
Title string
Expand Down Expand Up @@ -104,9 +93,7 @@ func (this *DownloadEnv) updateMedia(m3u8Url string, tsList []mformat.TsInfo) (e
}
if ts.Key.Method != `` {
keyContent, ok := uriToContentMap[ts.Key.KeyURI]
if ok {
ts.Key.KeyContent = keyContent
} else {
if ok == false {
var keyUrl string
keyUrl, errMsg = ResolveRefUrl(m3u8Url, ts.Key.KeyURI)
if errMsg != "" {
Expand All @@ -123,9 +110,9 @@ func (this *DownloadEnv) updateMedia(m3u8Url string, tsList []mformat.TsInfo) (e
if ts.Key.Method == mformat.EncryptMethod_AES128 && len(keyContent) != 16 { // Aes 128
return errors.New("invalid key " + strconv.Quote(string(keyContent)))
}
ts.Key.KeyContent = keyContent
uriToContentMap[ts.Key.KeyURI] = keyContent
}
ts.Key.KeyContent = keyContent
}
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion m3u8d-qt/mainwindow.ui
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
<item row="3" column="1">
<widget class="QLineEdit" name="lineEdit_SkipTsExpr">
<property name="placeholderText">
<string>1,92-100,http.code=403,if-http.code-merge_ts,time:00:05:12-00:07:20,with-skip_log</string>
<string>1,92-100,http.code=403,if-http.code-merge_ts,time:00:05:12-00:07:20</string>
</property>
</widget>
</item>
Expand Down
2 changes: 2 additions & 0 deletions mformat/ts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

// TsInfo 用于保存 ts 文件的下载地址和文件名
type TsInfo struct {
Idx uint32 // 从1开始, 每个ts增加1
Name string
Url string // 后续填充
URI string
Expand Down Expand Up @@ -93,6 +94,7 @@ func (this *M3U8File) GetTsList() (list []TsInfo) {
var seg = part.Segment
index++
var info = TsInfo{
Idx: uint32(index),
Name: fmt.Sprintf("%05d.ts", index), // ts视频片段命名规则
URI: seg.URI,
Url: "", // 之后填充
Expand Down
224 changes: 135 additions & 89 deletions skip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
Expand All @@ -19,15 +18,17 @@ import (
const SkipTimeSecEnd = 99 * 60 * 60

type SkipTsUnit struct {
Start uint32 // 包含
End uint32 // 包含
Start uint32 // 包含
End uint32 // 包含
OriginExpr string // 原始表达式
}

type SkipTsInfo struct {
HttpCodeList []int
SkipByIdxList []SkipTsUnit
IfHttpCodeMergeTs bool
SkipByTimeSecList []SkipTsUnit
KeepByTimeSecList []SkipTsUnit
}

func ParseSkipTsExpr(expr string) (info SkipTsInfo, errMsg string) {
Expand All @@ -50,19 +51,21 @@ func ParseSkipTsExpr(expr string) (info SkipTsInfo, errMsg string) {
i, err := strconv.Atoi(groups[1])
if err == nil || i > 0 {
ok = true
info.SkipByIdxList = skipListAddUnit(info.SkipByIdxList, SkipTsUnit{
Start: uint32(i),
End: uint32(i),
info.SkipByIdxList = append(info.SkipByIdxList, SkipTsUnit{
Start: uint32(i),
End: uint32(i),
OriginExpr: one,
})
}
} else if groups = betweenRe.FindStringSubmatch(one); len(groups) > 0 {
i1, err1 := strconv.Atoi(groups[1])
i2, err2 := strconv.Atoi(groups[2])
if err1 == nil && err2 == nil && i1 > 0 && i2 > 0 && i1 <= i2 {
ok = true
info.SkipByIdxList = skipListAddUnit(info.SkipByIdxList, SkipTsUnit{
Start: uint32(i1),
End: uint32(i2),
info.SkipByIdxList = append(info.SkipByIdxList, SkipTsUnit{
Start: uint32(i1),
End: uint32(i2),
OriginExpr: one,
})
}
} else if groups = httpCodeRe.FindStringSubmatch(one); len(groups) > 0 {
Expand All @@ -82,19 +85,17 @@ func ParseSkipTsExpr(expr string) (info SkipTsInfo, errMsg string) {
if err1 == nil && err2 == nil && startSec < endSec {
if groups[1] == "time" {
ok = true
info.SkipByTimeSecList = skipListAddUnit(info.SkipByTimeSecList, SkipTsUnit{
Start: startSec,
End: endSec,
info.SkipByTimeSecList = append(info.SkipByTimeSecList, SkipTsUnit{
Start: startSec,
End: endSec,
OriginExpr: one,
})
} else if groups[1] == "!time" {
ok = true
info.SkipByTimeSecList = skipListAddUnit(info.SkipByTimeSecList, SkipTsUnit{
Start: 0,
End: startSec,
})
info.SkipByTimeSecList = skipListAddUnit(info.SkipByTimeSecList, SkipTsUnit{
Start: endSec,
End: SkipTimeSecEnd,
info.KeepByTimeSecList = append(info.KeepByTimeSecList, SkipTsUnit{
Start: startSec,
End: endSec,
OriginExpr: one,
})
}
}
Expand All @@ -103,14 +104,19 @@ func ParseSkipTsExpr(expr string) (info SkipTsInfo, errMsg string) {
return info, "parse expr part invalid " + strconv.Quote(one)
}
}
sort.Slice(info.SkipByIdxList, func(i, j int) bool {
a, b := info.SkipByIdxList[i], info.SkipByIdxList[j]
return a.Start < b.Start
})
sort.Ints(info.HttpCodeList)
return info, ""
}

func (this *SkipTsUnit) IsCoverageFull(begin float64, end float64) bool {
return float64(this.Start) <= begin && float64(this.End) >= end
}

func (this *SkipTsUnit) HasIntersect(begin float64, end float64) bool {
newBegin := math.Max(begin, float64(this.Start))
newEnd := math.Min(end, float64(this.End))
return newBegin <= newEnd
}

func getTimeSecFromStr(str string) (sec uint32, err error) {
var h, m, s uint32

Expand All @@ -125,44 +131,6 @@ func getTimeSecFromStr(str string) (sec uint32, err error) {
return sec, nil
}

func maxUint32(a uint32, b uint32) uint32 {
if a > b {
return a
}
return b
}

func minUint32(a uint32, b uint32) uint32 {
if a > b {
return b
}
return a
}

func skipListAddUnit(skipList []SkipTsUnit, unit SkipTsUnit) (after []SkipTsUnit) {
for idx, one := range skipList {
// 交集的开始索引
jStart := maxUint32(one.Start, unit.Start)
// 交集的结束索引
jEnd := minUint32(one.End, unit.End)
// 有交集, 或者正好拼接为一个大区间10-20,21-30 => 10-30
if jStart <= jEnd || jStart == jEnd-1 {
unit.Start = minUint32(one.Start, unit.Start)
unit.End = maxUint32(one.End, unit.End)
var pre, post []SkipTsUnit // 前面部分，后面部分
pre = skipList[:idx]
if len(skipList) > idx+1 {
post = skipList[idx+1:]
}
skipList = append(pre, post...)
return skipListAddUnit(skipList, unit)
}
}
// 都无交集
skipList = append(skipList, unit)
return skipList
}

func isSkipByTsTime(beginSec float64, endSec float64, list []SkipTsUnit) bool {
for _, unit := range list {
newBegin := math.Max(float64(unit.Start), beginSec)
Expand All @@ -175,42 +143,120 @@ func isSkipByTsTime(beginSec float64, endSec float64, list []SkipTsUnit) bool {
return false
}

func skipApplyFilter(list []mformat.TsInfo, skipInfo SkipTsInfo) (after []mformat.TsInfo, skipList []mformat.TsInfo) {
var hasEmptyExtinf bool
for _, ts := range list {
if ts.TimeSec < 1e-5 {
hasEmptyExtinf = true
}
}
isSkipByTsIndex := func(idx uint32) bool {
for _, unit := range skipInfo.SkipByIdxList {
if unit.Start <= idx && idx <= unit.End {
return true
type skipFilterRecord struct {
ts mformat.TsInfo
reason string
}

// skipApplyFilter
//
// 策略:
// 确认每个ts文件都解析到了持续时常。
// 如果存在没有未解析到时间的ts文件，则不应用"按时间保留"、"按时间跳过"的规则
// 否则
// 1. 应用"按时间保留"规则, 把不需要保留的都剔除
// 2. 应用"按时间跳过"规则, 把需要跳过的都剔除
// 应用按照编号跳过的规则
func skipApplyFilter(list []mformat.TsInfo, skipInfo SkipTsInfo) (after []mformat.TsInfo, skipList []skipFilterRecord) {
timeRange, ok := calculateTsTimeRange(list)

if ok {
//应用"按时间保留"规则, 把不需要保留的都剔除
if len(skipInfo.KeepByTimeSecList) > 0 {
var keepIdxList = make([]bool, len(list)) // 每个是否保留

for _, rule := range skipInfo.KeepByTimeSecList {
for idx, tsTime := range timeRange {
if keepIdxList[idx] {
continue
}
//规则只要覆盖到了ts, 就要保留
if rule.HasIntersect(tsTime.begin, tsTime.end) {
keepIdxList[idx] = true
}
}
}
var newList []mformat.TsInfo
for idx, keep := range keepIdxList {
if keep == false {
skipList = append(skipList, skipFilterRecord{
ts: list[idx],
reason: "不在!time指明的时间范围内",
})
} else {
newList = append(newList, list[idx])
}
}
list = newList
}
return false
}

var timeBegin float64
var timeEnd float64
//应用"按时间跳过"规则, 把需要跳过的都剔除
if len(skipInfo.SkipByTimeSecList) > 0 {
var newList []mformat.TsInfo

for idx, ts := range list {
if idx > 0 {
timeBegin += list[idx-1].TimeSec
for idx, tsTime := range timeRange {
var match = false
for _, rule := range skipInfo.SkipByTimeSecList {
if rule.IsCoverageFull(tsTime.begin, tsTime.end) {
skipList = append(skipList, skipFilterRecord{
ts: list[idx],
reason: "匹配表达式" + rule.OriginExpr,
})
match = true
break
}
}
if match == false {
newList = append(newList, list[idx])
}
}
list = newList
}
timeEnd += ts.TimeSec
}

if isSkipByTsIndex(uint32(idx) + 1) {
skipList = append(skipList, ts)
continue
if len(skipInfo.SkipByIdxList) > 0 {
var newList []mformat.TsInfo
for _, ts := range list {
var match = false
for _, rule := range skipInfo.SkipByIdxList {
if rule.Start <= ts.Idx && ts.Idx <= rule.End {
skipList = append(skipList, skipFilterRecord{
ts: ts,
reason: "匹配表达式" + rule.OriginExpr,
})
match = true
break
}
}
if match == false {
newList = append(newList, ts)
}
}
if hasEmptyExtinf == false && isSkipByTsTime(timeBegin, timeEnd, skipInfo.SkipByTimeSecList) {
skipList = append(skipList, ts)
continue
list = newList
}
return list, skipList
}

type tsTimeRangeUnit struct {
begin float64
end float64
}

func calculateTsTimeRange(list []mformat.TsInfo) (timeRangeList []tsTimeRangeUnit, ok bool) {
var beginTime float64
for _, ts := range list {
//有未解析出持续时长的ts片段
if ts.TimeSec < 1e-5 {
return nil, false
}
after = append(after, ts)
endTime := beginTime + ts.TimeSec
timeRangeList = append(timeRangeList, tsTimeRangeUnit{
begin: beginTime,
end: endTime,
})
beginTime = endTime
}
return after, skipList
return timeRangeList, true
}

type removeSkipListResp struct {
Expand Down
Loading

0 comments on commit 4ba0f3f

Please sign in to comment.