Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

optimize: DNS获取到的IP，如果测速未通过，则不使用该IP，直接使用域名发起请求。 #333

Merged
merged 4 commits into from
Aug 20, 2024
Merged

optimize: DNS获取到的IP，如果测速未通过，则不使用该IP，直接使用域名发起请求。 #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5919441
feature: DNS获取到的IP，如果测速未通过，则不使用该IP，尝试使用域名发起请求。
wangliang181230 Aug 20, 2024
2b8acc2
代理连接超时时，关闭客户端连接。
wangliang181230 Aug 20, 2024
f8b0942
代理连接超时，也记录失败次数。
wangliang181230 Aug 20, 2024
da75399
代理连接超时，也记录失败次数。
wangliang181230 Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 23 additions & 32 deletions packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ const url = require('url')
const log = require('../../../utils/util.log')
const DnsUtil = require('../../dns/index')
const localIP = '127.0.0.1'
const defaultDns = require('dns')
// const matchUtil = require('../../../utils/util.match')
const speedTest = require('../../speed/index.js')
const dnsLookup = require('./dnsLookup')

function isSslConnect (sslConnectInterceptors, req, cltSocket, head) {
for (const intercept of sslConnectInterceptors) {
const ret = intercept(req, cltSocket, head)
Expand Down Expand Up @@ -36,10 +35,10 @@ module.exports = function createConnectHandler (sslConnectInterceptor, middlewar
if (isSslConnect(sslConnectInterceptors, req, cltSocket, head)) {
// 需要拦截，代替目标服务器，让客户端连接DS在本地启动的代理服务
fakeServerCenter.getServerPromise(hostname, port).then((serverObj) => {
log.info('--- fakeServer connect', hostname)
log.info(`----- fakeServer connect: ${localIP}:${serverObj.port} ➜ ${req.url} -----`)
connect(req, cltSocket, head, localIP, serverObj.port)
}, (e) => {
log.error('getServerPromise', e)
log.error(`----- fakeServer getServerPromise error: ${hostname}:${port}, error:`, e)
})
} else {
log.info(`未匹配到任何 sslConnectInterceptors，不拦截请求，直接连接目标服务器: ${hostname}:${port}`)
Expand All @@ -52,7 +51,7 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig/* , sniRegexpM
// tunneling https
// log.info('connect:', hostname, port)
const start = new Date()
let isDnsIntercept = null
const isDnsIntercept = {}
const hostport = `${hostname}:${port}`
// const replaceSni = matchUtil.matchHostname(sniRegexpMap, hostname, 'sni')
try {
Expand All @@ -61,30 +60,10 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig/* , sniRegexpM
host: hostname,
connectTimeout: 10000
}
if (dnsConfig) {
if (dnsConfig && dnsConfig.providers) {
const dns = DnsUtil.hasDnsLookup(dnsConfig, hostname)
if (dns) {
options.lookup = (hostname, options, callback) => {
const tester = speedTest.getSpeedTester(hostname)
if (tester) {
const aliveIpObj = tester.pickFastAliveIpObj()
if (aliveIpObj) {
log.info(`----- connect: ${hostport}, use alive ip from dns '${aliveIpObj.dns}': ${aliveIpObj.host} -----`)
callback(null, aliveIpObj.host, 4)
return
}
}
dns.lookup(hostname).then(ip => {
isDnsIntercept = { dns, hostname, ip }
if (ip !== hostname) {
log.info(`---- connect: ${hostport}, use ip from dns '${dns.name}': ${ip} ----`)
callback(null, ip, 4)
} else {
log.info(`----- connect: ${hostport}, use hostname: ${hostname} -----`)
defaultDns.lookup(hostname, options, callback)
}
})
}
options.lookup = dnsLookup.createLookupFunc(dns, 'connect', hostport, isDnsIntercept)
}
}
const proxySocket = net.connect(options, () => {
Expand All @@ -105,17 +84,29 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig/* , sniRegexpM
})
proxySocket.on('timeout', () => {
const cost = new Date() - start
log.info('代理socket timeout：', hostname, port, cost + 'ms')
const errorMsg = `代理连接超时: ${hostport}, cost: ${cost} ms`
log.error(errorMsg)

cltSocket.destroy()

if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error(`记录ip失败次数，用于优选ip！ hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.name}`)
}
})
proxySocket.on('error', (e) => {
// 连接失败，可能被GFW拦截，或者服务端拥挤
const cost = new Date() - start
log.error('代理连接失败：', e.message, hostname, port, cost + 'ms')
const errorMsg = `代理连接失败: ${hostport}, cost: ${cost} ms, errorMsg: ${e.message}`
log.error(errorMsg)

cltSocket.destroy()
if (isDnsIntercept) {

if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
log.error(`记录ip失败次数，用于优选ip！ hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.name}`)
}
})
return proxySocket
Expand Down
105 changes: 59 additions & 46 deletions packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ const DnsUtil = require('../../dns/index')
const log = require('../../../utils/util.log')
const RequestCounter = require('../../choice/RequestCounter')
const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware')
const speedTest = require('../../speed/index.js')
const defaultDns = require('dns')
const dnsLookup = require('./dnsLookup')
const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了

// create requestHandler function
module.exports = function createRequestHandler (createIntercepts, middlewares, externalProxy, dnsConfig, setting) {
// return
Expand Down Expand Up @@ -81,15 +81,15 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
}

function countSlow (isDnsIntercept, reason) {
if (isDnsIntercept) {
if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip, reason)
log.error(`记录ip失败次数，用于优选ip！ hostname: ${hostname}, ip: ${ip}, reason: ${reason}, dns: ${dns.name}`)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录proxy失败次数：', counter.value, reason)
log.error(`记录Proxy请求失败次数，用于切换备选域名！ hostname: ${counter.value}, reason: ${reason}, counter.count:`, counter.count)
}
}

Expand All @@ -108,35 +108,22 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
onFree()

function onFree () {
const url = `${rOptions.protocol}//${rOptions.hostname}:${rOptions.port}${rOptions.path}`
const url = `${rOptions.method} ➜ ${rOptions.protocol}//${rOptions.hostname}:${rOptions.port}${rOptions.path}`
const start = new Date()
log.info('代理请求:', url, rOptions.method, rOptions.servername ? ', sni: ' + rOptions.servername : '')
let isDnsIntercept
if (dnsConfig) {
const dns = DnsUtil.hasDnsLookup(dnsConfig, rOptions.hostname)
if (dns) {
rOptions.lookup = (hostname, options, callback) => {
const tester = speedTest.getSpeedTester(hostname)
if (tester) {
const aliveIpObj = tester.pickFastAliveIpObj()
if (aliveIpObj) {
log.info(`----- request url: ${url}, use alive ip from dns '${aliveIpObj.dns}': ${aliveIpObj.host} -----`)
callback(null, aliveIpObj.host, 4)
return
}
}
dns.lookup(hostname).then(ip => {
isDnsIntercept = { dns, hostname, ip }
if (ip !== hostname) {
log.info(`---- request url: ${url}, use ip from dns '${dns.name}': ${ip} ----`)
callback(null, ip, 4)
} else {
log.info(`---- request url: ${url}, use hostname: ${hostname} ----`)
defaultDns.lookup(hostname, options, callback)
}
})
log.info('发起代理请求:', url, (rOptions.servername ? ', sni: ' + rOptions.servername : ''))

const isDnsIntercept = {}
if (dnsConfig && dnsConfig.providers) {
let dns = DnsUtil.hasDnsLookup(dnsConfig, rOptions.hostname)
if (!dns && rOptions.servername) {
dns = dnsConfig.providers.quad9
if (dns) {
log.info(`域名 ${rOptions.hostname} 在dns中未配置，但使用了 sni: ${rOptions.servername}, 必须使用dns，现默认使用 'quad9' DNS.`)
}
}
if (dns) {
rOptions.lookup = dnsLookup.createLookupFunc(dns, 'request url', url, isDnsIntercept)
}
}

// rOptions.sigalgs = 'RSA-PSS+SHA256:RSA-PSS+SHA512:ECDSA+SHA256'
Expand All @@ -149,62 +136,73 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => {
const cost = new Date() - start
if (rOptions.protocol === 'https:') {
log.info('代理请求返回:', url, cost + 'ms')
log.info(`代理请求返回: ${url}, cost: ${cost} ms`)
} else {
log.info(`请求返回: ${url}, cost: ${cost} ms`)
}
// console.log('request:', proxyReq, proxyReq.socket)

if (cost > MAX_SLOW_TIME) {
countSlow(isDnsIntercept, 'to slow ' + cost + 'ms')
countSlow(isDnsIntercept, `代理请求成功但太慢, cost: ${cost} ms > ${MAX_SLOW_TIME} ms`)
}

resolve(proxyRes)
})

// 代理请求的事件监听
proxyReq.on('timeout', () => {
const cost = new Date() - start
log.error('代理请求超时', rOptions.protocol, rOptions.hostname, rOptions.path, cost + 'ms')
countSlow(isDnsIntercept, 'to slow ' + cost + 'ms')
const errorMsg = `代理请求超时: ${url}, cost: ${cost} ms`
log.error(errorMsg)
countSlow(isDnsIntercept, `代理请求超时, cost: ${cost} ms`)
proxyReq.end()
proxyReq.destroy()
const error = new Error(`${rOptions.host}:${rOptions.port}, 代理请求超时`)
const error = new Error(errorMsg)
error.status = 408
reject(error)
})
proxyReq.on('error', (e) => {
const cost = new Date() - start
log.error('代理请求错误', e.code, e.message, rOptions.hostname, rOptions.path, cost + 'ms')
countSlow(isDnsIntercept, 'error:' + e.message)
log.error(`代理请求错误: ${url}, cost: ${cost} ms, error:`, e)
countSlow(isDnsIntercept, '代理请求错误: ' + e.message)
reject(e)
})
proxyReq.on('aborted', () => {
const cost = new Date() - start
log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms')
const errorMsg = `代理请求被取消: ${url}, cost: ${cost} ms`
log.error(errorMsg)

if (cost > MAX_SLOW_TIME) {
countSlow(isDnsIntercept, 'to slow ' + cost + 'ms')
countSlow(isDnsIntercept, `代理请求被取消，且请求太慢, cost: ${cost} ms > ${MAX_SLOW_TIME} ms`)
}

if (res.writableEnded) {
return
}
reject(new Error('代理请求被取消'))
reject(new Error(errorMsg))
})

// 原始请求的事件监听
req.on('aborted', function () {
log.error('请求被取消', rOptions.hostname, rOptions.path)
const cost = new Date() - start
const errorMsg = `请求被取消: ${url}, cost: ${cost} ms`
log.error(errorMsg)
proxyReq.abort()
if (res.writableEnded) {
return
}
reject(new Error('请求被取消'))
reject(new Error(errorMsg))
})
req.on('error', function (e, req, res) {
log.error('请求错误：', e.errno, rOptions.hostname, rOptions.path)
const cost = new Date() - start
log.error(`请求错误: ${url}, cost: ${cost} ms, error:`, e)
reject(e)
})
req.on('timeout', () => {
log.error('请求超时', rOptions.hostname, rOptions.path)
reject(new Error(`${rOptions.hostname}:${rOptions.port}, 请求超时`))
const cost = new Date() - start
const errorMsg = `请求超时: ${url}, cost: ${cost} ms`
log.error(errorMsg)
reject(new Error(errorMsg))
})
req.pipe(proxyReq)
}
Expand Down Expand Up @@ -318,6 +316,21 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
// do nothing
}

// region 忽略部分已经打印过ERROR日志的错误
if (e.message) {
const ignoreErrors = [
'代理请求错误: ',
'代理请求超时: ',
'代理请求被取消: '
]
for (const ignoreError of ignoreErrors) {
if (e.message.startsWith(ignoreError)) {
return
}
}
}
// endregion

log.error('Request error:', e)
}
})
Expand Down
56 changes: 56 additions & 0 deletions packages/mitmproxy/src/lib/proxy/mitmproxy/dnsLookup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
const speedTest = require('../../speed')
const log = require('../../../utils/util.log')
const defaultDns = require('dns')

module.exports = {
createLookupFunc: function (dns, action, target, isDnsIntercept) {
return (hostname, options, callback) => {
const tester = speedTest.getSpeedTester(hostname)
if (tester && tester.ready) {
const aliveIpObj = tester.pickFastAliveIpObj()
if (aliveIpObj) {
log.info(`----- ${action}: ${target}, use alive ip from dns '${aliveIpObj.dns}': ${aliveIpObj.host} -----`)
callback(null, aliveIpObj.host, 4)
return
} else {
log.info(`----- ${action}: ${target}, no alive ip, tester:`, tester)
}
}
dns.lookup(hostname).then(ip => {
if (isDnsIntercept) {
isDnsIntercept.dns = dns
isDnsIntercept.hostname = hostname
isDnsIntercept.ip = ip
}

if (ip !== hostname) {
// 判断是否为测速失败的IP，如果是，则不使用当前IP
let isTestFailedIp = false
if (tester && tester.ready && tester.backupList && tester.backupList.length > 0) {
for (let i = 0; i < tester.backupList.length; i++) {
const item = tester.backupList[i]
if (item.host === ip) {
if (item.time == null) {
isTestFailedIp = true
}
break
}
}
}
if (isTestFailedIp === false) {
log.info(`----- ${action}: ${target}, use ip from dns '${dns.name}': ${ip} -----`)
callback(null, ip, 4)
return
} else {
// 使用默认dns
log.info(`----- ${action}: ${target}, use hostname by default DNS: ${hostname}, skip test failed ip from dns '${dns.name}: ${ip}', options:`, options)
}
} else {
// 使用默认dns
log.info(`----- ${action}: ${target}, use hostname by default DNS: ${hostname}, options:`, options, ', dns:', dns)
}
defaultDns.lookup(hostname, options, callback)
})
}
}
}